openstack
/
nova
Code Issues Proposed changes

'limit' and 'marker' support for db_api and keypair_obj 

keypair object and db_api supports 'limit' and 'marker'
parameters, these changes are required for keypairs pagination
for Nova API.

Part of blueprint keypairs-pagination

Change-Id: I9776efc609f01d053824b31f64126cfcd6dadc18
This commit is contained in:
Pavel Kholkin 2016-07-06 14:45:35 +03:00
parent d219e25452
commit 777386b4bf
9 changed files with 352 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nova/db/api.py
View File

@ -964,9 +964,10 @@ def key_pair_get(context, user_id, name):
return IMPL.key_pair_get(context, user_id, name)
def key_pair_get_all_by_user(context, user_id):
def key_pair_get_all_by_user(context, user_id, limit=None, marker=None):
"""Get all key_pairs by user."""
return IMPL.key_pair_get_all_by_user(context, user_id)
return IMPL.key_pair_get_all_by_user(
context, user_id, limit=limit, marker=marker)
def key_pair_count_by_user(context, user_id):

19
nova/db/sqlalchemy/api.py
View File

@ -2929,10 +2929,21 @@ def key_pair_get(context, user_id, name):
@require_context
@main_context_manager.reader
def key_pair_get_all_by_user(context, user_id):
return model_query(context, models.KeyPair, read_deleted="no").\
filter_by(user_id=user_id).\
all()
def key_pair_get_all_by_user(context, user_id, limit=None, marker=None):
marker_row = None
if marker is not None:
marker_row = model_query(context, models.KeyPair, read_deleted="no").\
filter_by(name=marker).filter_by(user_id=user_id).first()
if not marker_row:
raise exception.MarkerNotFound(marker=marker)
query = model_query(context, models.KeyPair, read_deleted="no").\
filter_by(user_id=user_id)
query = sqlalchemyutils.paginate_query(
query, models.KeyPair, limit, ['name'], marker=marker_row)
return query.all()
@require_context

50
nova/objects/keypair.py
View File

@ -13,6 +13,7 @@
# under the License.
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import utils as sqlalchemyutils
from oslo_log import log as logging
from oslo_utils import versionutils
@ -32,7 +33,7 @@ LOG = logging.getLogger(__name__)
@db_api.api_context_manager.reader
def _get_from_db(context, user_id, name=None):
def _get_from_db(context, user_id, name=None, limit=None, marker=None):
query = context.session.query(api_models.KeyPair).\
filter(api_models.KeyPair.user_id == user_id)
if name is not None:
@ -41,8 +42,19 @@ def _get_from_db(context, user_id, name=None):
if not db_keypair:
raise exception.KeypairNotFound(user_id=user_id, name=name)
return db_keypair
else:
return query.all()
marker_row = None
if marker is not None:
marker_row = context.session.query(api_models.KeyPair).\
filter(api_models.KeyPair.name == marker).\
filter(api_models.KeyPair.user_id == user_id).first()
if not marker_row:
raise exception.MarkerNotFound(marker=marker)
query = sqlalchemyutils.paginate_query(
query, api_models.KeyPair, limit, ['name'], marker=marker_row)
return query.all()
@db_api.api_context_manager.reader
@ -180,24 +192,44 @@ class KeyPairList(base.ObjectListBase, base.NovaObject):
# KeyPair <= version 1.1
# Version 1.1: KeyPair <= version 1.2
# Version 1.2: KeyPair <= version 1.3
VERSION = '1.2'
# Version 1.3: Add new parameters 'limit' and 'marker' to get_by_user()
VERSION = '1.3'
fields = {
'objects': fields.ListOfObjectsField('KeyPair'),
}
@staticmethod
def _get_from_db(context, user_id):
return _get_from_db(context, user_id)
def _get_from_db(context, user_id, limit, marker):
return _get_from_db(context, user_id, limit=limit, marker=marker)
@staticmethod
def _get_count_from_db(context, user_id):
return _get_count_from_db(context, user_id)
@base.remotable_classmethod
def get_by_user(cls, context, user_id):
api_db_keypairs = cls._get_from_db(context, user_id)
main_db_keypairs = db.key_pair_get_all_by_user(context, user_id)
def get_by_user(cls, context, user_id, limit=None, marker=None):
try:
api_db_keypairs = cls._get_from_db(
context, user_id, limit=limit, marker=marker)
# NOTE(pkholkin): If we were asked for a marker and found it in
# results from the API DB, we must continue our pagination with
# just the limit (if any) to the main DB.
marker = None
except exception.MarkerNotFound:
api_db_keypairs = []
if limit is not None:
limit_more = limit - len(api_db_keypairs)
else:
limit_more = None
if limit_more is None or limit_more > 0:
main_db_keypairs = db.key_pair_get_all_by_user(
context, user_id, limit=limit_more, marker=marker)
else:
main_db_keypairs = []
return base.obj_make_list(context, cls(context), objects.KeyPair,
api_db_keypairs + main_db_keypairs)

109
nova/tests/functional/db/test_keypair.py
View File

@ -146,7 +146,9 @@ class KeyPairObjectTestCase(test.NoDBTestCase):
# NOTE(danms): This only fetches from the API DB
api_keys = objects.KeyPairList._get_from_db(self.context,
self.context.user_id)
self.context.user_id,
limit=None,
marker=None)
self.assertEqual(3, len(api_keys))
# NOTE(danms): This only fetches from the main DB
@ -177,3 +179,108 @@ class KeyPairObjectTestCase(test.NoDBTestCase):
total, done = keypair.migrate_keypairs_to_api_db(self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)
def test_get_by_user_limit_and_marker(self):
self._api_kp(name='apikey1')
self._api_kp(name='apikey2')
self._main_kp(name='mainkey1')
self._main_kp(name='mainkey2')
# check all 4 keypairs (2 api and 2 main)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id)
self.assertEqual(4, len(kpl))
self.assertEqual(set(['apikey1', 'apikey2', 'mainkey1', 'mainkey2']),
set([x.name for x in kpl]))
# check only 1 keypair (1 api)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id,
limit=1)
self.assertEqual(1, len(kpl))
self.assertEqual(set(['apikey1']),
set([x.name for x in kpl]))
# check only 3 keypairs (2 api and 1 main)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id,
limit=3)
self.assertEqual(3, len(kpl))
self.assertEqual(set(['apikey1', 'apikey2', 'mainkey1']),
set([x.name for x in kpl]))
# check keypairs after 'apikey1' (1 api and 2 main)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id,
marker='apikey1')
self.assertEqual(3, len(kpl))
self.assertEqual(set(['apikey2', 'mainkey1', 'mainkey2']),
set([x.name for x in kpl]))
# check keypairs after 'mainkey2' (no keypairs)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id,
marker='mainkey2')
self.assertEqual(0, len(kpl))
# check only 2 keypairs after 'apikey1' (1 api and 1 main)
kpl = objects.KeyPairList.get_by_user(self.context,
self.context.user_id,
limit=2,
marker='apikey1')
self.assertEqual(2, len(kpl))
self.assertEqual(set(['apikey2', 'mainkey1']),
set([x.name for x in kpl]))
# check non-existing keypair
self.assertRaises(exception.MarkerNotFound,
objects.KeyPairList.get_by_user,
self.context, self.context.user_id,
limit=2, marker='unknown_kp')
def test_get_by_user_different_users(self):
# create keypairs for two users
self._api_kp(name='apikey', user_id='user1')
self._api_kp(name='apikey', user_id='user2')
self._main_kp(name='mainkey', user_id='user1')
self._main_kp(name='mainkey', user_id='user2')
# check all 2 keypairs for user1 (1 api and 1 main)
kpl = objects.KeyPairList.get_by_user(self.context, 'user1')
self.assertEqual(2, len(kpl))
self.assertEqual(set(['apikey', 'mainkey']),
set([x.name for x in kpl]))
# check all 2 keypairs for user2 (1 api and 1 main)
kpl = objects.KeyPairList.get_by_user(self.context, 'user2')
self.assertEqual(2, len(kpl))
self.assertEqual(set(['apikey', 'mainkey']),
set([x.name for x in kpl]))
# check only 1 keypair for user1 (1 api)
kpl = objects.KeyPairList.get_by_user(self.context, 'user1', limit=1)
self.assertEqual(1, len(kpl))
self.assertEqual(set(['apikey']),
set([x.name for x in kpl]))
# check keypairs after 'apikey' for user2 (1 main)
kpl = objects.KeyPairList.get_by_user(self.context, 'user2',
marker='apikey')
self.assertEqual(1, len(kpl))
self.assertEqual(set(['mainkey']),
set([x.name for x in kpl]))
# check only 2 keypairs after 'apikey' for user1 (1 main)
kpl = objects.KeyPairList.get_by_user(self.context,
'user1',
limit=2,
marker='apikey')
self.assertEqual(1, len(kpl))
self.assertEqual(set(['mainkey']),
set([x.name for x in kpl]))
# check non-existing keypair for user2
self.assertRaises(exception.MarkerNotFound,
objects.KeyPairList.get_by_user,
self.context, 'user2',
limit=2, marker='unknown_kp')

2
nova/tests/unit/api/openstack/compute/test_keypairs.py
View File

@ -47,7 +47,7 @@ def fake_keypair(name):
name=name, **keypair_data)
def db_key_pair_get_all_by_user(self, user_id):
def db_key_pair_get_all_by_user(self, user_id, limit, marker):
return [fake_keypair('FAKE')]

2
nova/tests/unit/compute/test_keypairs.py
View File

@ -53,7 +53,7 @@ class KeypairAPITestCase(test_compute.BaseTestCase):
def _keypair_db_call_stubs(self):
def db_key_pair_get_all_by_user(context, user_id):
def db_key_pair_get_all_by_user(context, user_id, limit, marker):
return [dict(test_keypair.fake_keypair,
name=self.existing_key_name,
public_key=self.pub_key,

86
nova/tests/unit/db/test_db_api.py
View File

@ -6958,6 +6958,92 @@ class KeyPairTestCase(test.TestCase, ModelsObjectComparatorMixin):
self._assertEqualListsOfObjects(key_pairs_user_1, real_keys_1)
self._assertEqualListsOfObjects(key_pairs_user_2, real_keys_2)
def test_key_pair_get_all_by_user_limit_and_marker(self):
params = [
{'name': 'test_1', 'user_id': 'test_user_id', 'type': 'ssh'},
{'name': 'test_2', 'user_id': 'test_user_id', 'type': 'ssh'},
{'name': 'test_3', 'user_id': 'test_user_id', 'type': 'ssh'}
]
# check all 3 keypairs
keys = [self._create_key_pair(p) for p in params]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_id')
self._assertEqualListsOfObjects(keys, db_keys)
# check only 1 keypair
expected_keys = [keys[0]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_id',
limit=1)
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check keypairs after 'test_1'
expected_keys = [keys[1], keys[2]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_id',
marker='test_1')
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check only 1 keypairs after 'test_1'
expected_keys = [keys[1]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_id',
limit=1,
marker='test_1')
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check non-existing keypair
self.assertRaises(exception.MarkerNotFound,
db.key_pair_get_all_by_user,
self.ctxt, 'test_user_id',
limit=1, marker='unknown_kp')
def test_key_pair_get_all_by_user_different_users(self):
params1 = [
{'name': 'test_1', 'user_id': 'test_user_1', 'type': 'ssh'},
{'name': 'test_2', 'user_id': 'test_user_1', 'type': 'ssh'},
{'name': 'test_3', 'user_id': 'test_user_1', 'type': 'ssh'}
]
params2 = [
{'name': 'test_1', 'user_id': 'test_user_2', 'type': 'ssh'},
{'name': 'test_2', 'user_id': 'test_user_2', 'type': 'ssh'},
{'name': 'test_3', 'user_id': 'test_user_2', 'type': 'ssh'}
]
# create keypairs for two users
keys1 = [self._create_key_pair(p) for p in params1]
keys2 = [self._create_key_pair(p) for p in params2]
# check all 2 keypairs for test_user_1
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_1')
self._assertEqualListsOfObjects(keys1, db_keys)
# check all 2 keypairs for test_user_2
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_2')
self._assertEqualListsOfObjects(keys2, db_keys)
# check only 1 keypair for test_user_1
expected_keys = [keys1[0]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_1',
limit=1)
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check keypairs after 'test_1' for test_user_2
expected_keys = [keys2[1], keys2[2]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_2',
marker='test_1')
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check only 1 keypairs after 'test_1' for test_user_1
expected_keys = [keys1[1]]
db_keys = db.key_pair_get_all_by_user(self.ctxt, 'test_user_1',
limit=1,
marker='test_1')
self._assertEqualListsOfObjects(expected_keys, db_keys)
# check non-existing keypair for test_user_2
self.assertRaises(exception.MarkerNotFound,
db.key_pair_get_all_by_user,
self.ctxt, 'test_user_2',
limit=1, marker='unknown_kp')
def test_key_pair_count_by_user(self):
params = [
{'name': 'test_1', 'user_id': 'test_user_id_1', 'type': 'ssh'},

98
nova/tests/unit/objects/test_keypair.py
View File

@ -12,6 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import mock
from oslo_utils import timeutils
@ -118,9 +120,11 @@ class _TestKeyPairObject(object):
self.compare_obj(keypairs[1], fake_keypair)
self.assertEqual(2, keypair.KeyPairList.get_count_by_user(self.context,
'fake-user'))
mock_kp_get.assert_called_once_with(self.context, 'fake-user')
mock_kp_get.assert_called_once_with(self.context, 'fake-user',
limit=None, marker=None)
mock_kp_count.assert_called_once_with(self.context, 'fake-user')
mock_api_get.assert_called_once_with(self.context, 'fake-user')
mock_api_get.assert_called_once_with(self.context, 'fake-user',
limit=None, marker=None)
mock_api_count.assert_called_once_with(self.context, 'fake-user')
def test_obj_make_compatible(self):
@ -130,6 +134,96 @@ class _TestKeyPairObject(object):
keypair_obj.obj_make_compatible(fake_keypair_copy, '1.1')
self.assertNotIn('type', fake_keypair_copy)
@mock.patch('nova.db.key_pair_get_all_by_user')
@mock.patch('nova.objects.KeyPairList._get_from_db')
def test_get_by_user_limit(self, mock_api_get, mock_kp_get):
api_keypair = copy.deepcopy(fake_keypair)
api_keypair['name'] = 'api_kp'
mock_api_get.return_value = [api_keypair]
mock_kp_get.return_value = [fake_keypair]
keypairs = keypair.KeyPairList.get_by_user(self.context, 'fake-user',
limit=1)
self.assertEqual(1, len(keypairs))
self.compare_obj(keypairs[0], api_keypair)
mock_api_get.assert_called_once_with(self.context, 'fake-user',
limit=1, marker=None)
self.assertFalse(mock_kp_get.called)
@mock.patch('nova.db.key_pair_get_all_by_user')
@mock.patch('nova.objects.KeyPairList._get_from_db')
def test_get_by_user_marker(self, mock_api_get, mock_kp_get):
api_kp_name = 'api_kp'
mock_api_get.side_effect = exception.MarkerNotFound(marker=api_kp_name)
mock_kp_get.return_value = [fake_keypair]
keypairs = keypair.KeyPairList.get_by_user(self.context, 'fake-user',
marker=api_kp_name)
self.assertEqual(1, len(keypairs))
self.compare_obj(keypairs[0], fake_keypair)
mock_api_get.assert_called_once_with(self.context, 'fake-user',
limit=None,
marker=api_kp_name)
mock_kp_get.assert_called_once_with(self.context, 'fake-user',
limit=None,
marker=api_kp_name)
@mock.patch('nova.db.key_pair_get_all_by_user')
@mock.patch('nova.objects.KeyPairList._get_from_db')
def test_get_by_user_limit_and_marker_api(self, mock_api_get, mock_kp_get):
first_api_kp_name = 'first_api_kp'
api_keypair = copy.deepcopy(fake_keypair)
api_keypair['name'] = 'api_kp'
mock_api_get.return_value = [api_keypair]
mock_kp_get.return_value = [fake_keypair]
keypairs = keypair.KeyPairList.get_by_user(self.context, 'fake-user',
limit=5,
marker=first_api_kp_name)
self.assertEqual(2, len(keypairs))
self.compare_obj(keypairs[0], api_keypair)
self.compare_obj(keypairs[1], fake_keypair)
mock_api_get.assert_called_once_with(self.context, 'fake-user',
limit=5,
marker=first_api_kp_name)
mock_kp_get.assert_called_once_with(self.context, 'fake-user',
limit=4, marker=None)
@mock.patch('nova.db.key_pair_get_all_by_user')
@mock.patch('nova.objects.KeyPairList._get_from_db')
def test_get_by_user_limit_and_marker_main(self, mock_api_get,
mock_kp_get):
first_main_kp_name = 'first_main_kp'
mock_api_get.side_effect = exception.MarkerNotFound(
marker=first_main_kp_name)
mock_kp_get.return_value = [fake_keypair]
keypairs = keypair.KeyPairList.get_by_user(self.context, 'fake-user',
limit=5,
marker=first_main_kp_name)
self.assertEqual(1, len(keypairs))
self.compare_obj(keypairs[0], fake_keypair)
mock_api_get.assert_called_once_with(self.context, 'fake-user',
limit=5,
marker=first_main_kp_name)
mock_kp_get.assert_called_once_with(self.context, 'fake-user',
limit=5, marker=first_main_kp_name)
@mock.patch('nova.db.key_pair_get_all_by_user')
@mock.patch('nova.objects.KeyPairList._get_from_db')
def test_get_by_user_limit_and_marker_invalid_marker(
self, mock_api_get, mock_kp_get):
kp_name = 'unknown_kp'
mock_api_get.side_effect = exception.MarkerNotFound(marker=kp_name)
mock_kp_get.side_effect = exception.MarkerNotFound(marker=kp_name)
self.assertRaises(exception.MarkerNotFound,
keypair.KeyPairList.get_by_user,
self.context, 'fake-user',
limit=5, marker=kp_name)
class TestMigrationObject(test_objects._LocalTest,
_TestKeyPairObject):

2
nova/tests/unit/objects/test_objects.py
View File

@ -1157,7 +1157,7 @@ object_data = {
'LibvirtLiveMigrateBDMInfo': '1.0-252aabb723ca79d5469fa56f64b57811',
'LibvirtLiveMigrateData': '1.3-2795e5646ee21e8c7f1c3e64fb6c80a3',
'KeyPair': '1.4-1244e8d1b103cc69d038ed78ab3a8cc6',
'KeyPairList': '1.2-58b94f96e776bedaf1e192ddb2a24c4e',
'KeyPairList': '1.3-94aad3ac5c938eef4b5e83da0212f506',
'Migration': '1.4-17979b9f2ae7f28d97043a220b2a8350',
'MigrationContext': '1.1-9fb17b0b521370957a884636499df52d',
'MigrationList': '1.3-55595bfc1a299a5962614d0821a3567e',