diff --git a/nova/auth/users.py b/nova/auth/users.py index a45fbc1c0..bb78dad99 100644 --- a/nova/auth/users.py +++ b/nova/auth/users.py @@ -239,7 +239,6 @@ class Project(Group): def __init__(self, id, project_manager_id, description, member_ids): self.project_manager_id = project_manager_id super(Project, self).__init__(id, description, member_ids) - self.keeper = datastore.Keeper(prefix="project-") @property def project_manager(self): diff --git a/nova/datastore.py b/nova/datastore.py index 5a9b80c62..922432a23 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -21,19 +21,10 @@ """ Datastore: -Providers the Keeper class, a simple pseudo-dictionary that -persists on disk. - MAKE Sure that ReDIS is running, and your flags are set properly, before trying to run this. """ -import json -import logging -import os -import sqlite3 -import time - from nova import vendor import redis @@ -42,15 +33,11 @@ from nova import utils FLAGS = flags.FLAGS -flags.DEFINE_string('datastore_path', utils.abspath('../keeper'), - 'where keys are stored on disk') flags.DEFINE_string('redis_host', '127.0.0.1', 'Host that redis is running on.') flags.DEFINE_integer('redis_port', 6379, 'Port that redis is running on.') flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') -flags.DEFINE_string('keeper_backend', 'redis', - 'which backend to use for keeper') class Redis(object): @@ -61,247 +48,9 @@ class Redis(object): @classmethod def instance(cls): if not hasattr(cls, '_instance'): - inst = redis.Redis(host=FLAGS.redis_host, port=FLAGS.redis_port, db=FLAGS.redis_db) + inst = redis.Redis(host=FLAGS.redis_host, + port=FLAGS.redis_port, + db=FLAGS.redis_db) cls._instance = inst return cls._instance - -def slugify(key, prefix=None): - """ - Key has to be a valid filename. Slugify solves that. - """ - return "%s%s" % (prefix, key) - - -class SqliteKeeper(object): - """ Keeper implementation in SQLite, mostly for in-memory testing """ - _conn = {} # class variable - - def __init__(self, prefix): - self.prefix = prefix - - @property - def conn(self): - if self.prefix not in self.__class__._conn: - logging.debug('no sqlite connection (%s), making new', self.prefix) - if FLAGS.datastore_path != ':memory:': - try: - os.mkdir(FLAGS.datastore_path) - except Exception: - pass - conn = sqlite3.connect(os.path.join( - FLAGS.datastore_path, '%s.sqlite' % self.prefix)) - else: - conn = sqlite3.connect(':memory:') - - c = conn.cursor() - try: - c.execute('''CREATE TABLE data (item text, value text)''') - conn.commit() - except Exception: - logging.exception('create table failed') - finally: - c.close() - - self.__class__._conn[self.prefix] = conn - - return self.__class__._conn[self.prefix] - - def __delitem__(self, item): - #logging.debug('sqlite deleting %s', item) - c = self.conn.cursor() - try: - c.execute('DELETE FROM data WHERE item = ?', (item, )) - self.conn.commit() - except Exception: - logging.exception('delete failed: %s', item) - finally: - c.close() - - def __getitem__(self, item): - #logging.debug('sqlite getting %s', item) - result = None - c = self.conn.cursor() - try: - c.execute('SELECT value FROM data WHERE item = ?', (item, )) - row = c.fetchone() - if row: - result = json.loads(row[0]) - else: - result = None - except Exception: - logging.exception('select failed: %s', item) - finally: - c.close() - #logging.debug('sqlite got %s: %s', item, result) - return result - - def __setitem__(self, item, value): - serialized_value = json.dumps(value) - insert = True - if self[item] is not None: - insert = False - #logging.debug('sqlite insert %s: %s', item, value) - c = self.conn.cursor() - try: - if insert: - c.execute('INSERT INTO data VALUES (?, ?)', - (item, serialized_value)) - else: - c.execute('UPDATE data SET item=?, value=? WHERE item = ?', - (item, serialized_value, item)) - - self.conn.commit() - except Exception: - logging.exception('select failed: %s', item) - finally: - c.close() - - def clear(self): - if self.prefix not in self.__class__._conn: - return - self.conn.close() - if FLAGS.datastore_path != ':memory:': - os.unlink(os.path.join(FLAGS.datastore_path, '%s.sqlite' % self.prefix)) - del self.__class__._conn[self.prefix] - - def clear_all(self): - for k, conn in self.__class__._conn.iteritems(): - conn.close() - if FLAGS.datastore_path != ':memory:': - os.unlink(os.path.join(FLAGS.datastore_path, - '%s.sqlite' % self.prefix)) - self.__class__._conn = {} - - - def set_add(self, item, value): - group = self[item] - if not group: - group = [] - group.append(value) - self[item] = group - - def set_is_member(self, item, value): - group = self[item] - if not group: - return False - return value in group - - def set_remove(self, item, value): - group = self[item] - if not group: - group = [] - group.remove(value) - self[item] = group - - def set_members(self, item): - group = self[item] - if not group: - group = [] - return group - - def set_fetch(self, item): - # TODO(termie): I don't really know what set_fetch is supposed to do - group = self[item] - if not group: - group = [] - return iter(group) - -class JsonKeeper(object): - """ - Simple dictionary class that persists using - JSON in files saved to disk. - """ - def __init__(self, prefix): - self.prefix = prefix - - def __delitem__(self, item): - """ - Removing a key means deleting a file from disk. - """ - item = slugify(item, self.prefix) - path = "%s/%s" % (FLAGS.datastore_path, item) - if os.path.isfile(path): - os.remove(path) - - def __getitem__(self, item): - """ - Fetch file contents and dejsonify them. - """ - item = slugify(item, self.prefix) - path = "%s/%s" % (FLAGS.datastore_path, item) - if os.path.isfile(path): - return json.load(open(path, 'r')) - return None - - def __setitem__(self, item, value): - """ - JSON encode value and save to file. - """ - item = slugify(item, self.prefix) - path = "%s/%s" % (FLAGS.datastore_path, item) - with open(path, "w") as blobfile: - blobfile.write(json.dumps(value)) - return value - - -class RedisKeeper(object): - """ - Simple dictionary class that persists using - ReDIS. - """ - def __init__(self, prefix="redis-"): - self.prefix = prefix - #Redis.instance().ping() - - def __setitem__(self, item, value): - """ - JSON encode value and save to file. - """ - item = slugify(item, self.prefix) - Redis.instance().set(item, json.dumps(value)) - return value - - def __getitem__(self, item): - item = slugify(item, self.prefix) - value = Redis.instance().get(item) - if value: - return json.loads(value) - - def __delitem__(self, item): - item = slugify(item, self.prefix) - return Redis.instance().delete(item) - - def clear(self): - raise NotImplementedError() - - def clear_all(self): - raise NotImplementedError() - - def set_add(self, item, value): - item = slugify(item, self.prefix) - return Redis.instance().sadd(item, json.dumps(value)) - - def set_is_member(self, item, value): - item = slugify(item, self.prefix) - return Redis.instance().sismember(item, json.dumps(value)) - - def set_remove(self, item, value): - item = slugify(item, self.prefix) - return Redis.instance().srem(item, json.dumps(value)) - - def set_members(self, item): - item = slugify(item, self.prefix) - return [json.loads(v) for v in Redis.instance().smembers(item)] - - def set_fetch(self, item): - item = slugify(item, self.prefix) - for obj in Redis.instance().sinter([item]): - yield json.loads(obj) - - -def Keeper(prefix=''): - KEEPERS = {'redis': RedisKeeper, - 'sqlite': SqliteKeeper} - return KEEPERS[FLAGS.keeper_backend](prefix) - diff --git a/nova/test.py b/nova/test.py index c226b43e7..fa05a02af 100644 --- a/nova/test.py +++ b/nova/test.py @@ -36,7 +36,6 @@ from twisted.python import failure from twisted.trial import unittest as trial_unittest import stubout -from nova import datastore from nova import fakerabbit from nova import flags @@ -79,10 +78,6 @@ class TrialTestCase(trial_unittest.TestCase): if FLAGS.fake_rabbit: fakerabbit.reset_all() - # attempt to wipe all keepers - #keeper = datastore.Keeper() - #keeper.clear_all() - def flags(self, **kw): for k, v in kw.iteritems(): if k in self.flag_overrides: diff --git a/nova/tests/datastore_unittest.py b/nova/tests/datastore_unittest.py deleted file mode 100644 index f4165a257..000000000 --- a/nova/tests/datastore_unittest.py +++ /dev/null @@ -1,80 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2010 Anso Labs, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import test -from nova import datastore -import random - -class KeeperTestCase(test.BaseTestCase): - """ - Basic persistence tests for Keeper datastore. - Generalize, then use these to support - migration to redis / cassandra / multiple stores. - """ - - def __init__(self, *args, **kwargs): - """ - Create a new keeper instance for test keys. - """ - super(KeeperTestCase, self).__init__(*args, **kwargs) - self.keeper = datastore.Keeper('test-') - - def tear_down(self): - """ - Scrub out test keeper data. - """ - pass - - def test_store_strings(self): - """ - Confirm that simple strings go in and come out safely. - Should also test unicode strings. - """ - randomstring = ''.join( - [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') - for _x in xrange(20)] - ) - self.keeper['test_string'] = randomstring - self.assertEqual(randomstring, self.keeper['test_string']) - - def test_store_dicts(self): - """ - Arbitrary dictionaries should be storable. - """ - test_dict = {'key_one': 'value_one'} - self.keeper['test_dict'] = test_dict - self.assertEqual(test_dict['key_one'], - self.keeper['test_dict']['key_one']) - - def test_sets(self): - """ - A keeper dict should be self-serializing. - """ - self.keeper.set_add('test_set', 'foo') - test_dict = {'arbitrary': 'dict of stuff'} - self.keeper.set_add('test_set', test_dict) - self.assertTrue(self.keeper.set_is_member('test_set', 'foo')) - self.assertFalse(self.keeper.set_is_member('test_set', 'bar')) - self.keeper.set_remove('test_set', 'foo') - self.assertFalse(self.keeper.set_is_member('test_set', 'foo')) - rv = self.keeper.set_fetch('test_set') - self.assertEqual(test_dict, rv.next()) - self.keeper.set_remove('test_set', test_dict) - diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index d40172cfc..cbc0042da 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -27,6 +27,4 @@ FLAGS.fake_storage = True FLAGS.fake_rabbit = True FLAGS.fake_network = True FLAGS.fake_users = True -#FLAGS.keeper_backend = 'sqlite' -FLAGS.datastore_path = ':memory:' FLAGS.verbose = True diff --git a/nova/tests/keeper_unittest.py b/nova/tests/keeper_unittest.py deleted file mode 100644 index 0a993afd7..000000000 --- a/nova/tests/keeper_unittest.py +++ /dev/null @@ -1,76 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2010 Anso Labs, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import random - -from nova import datastore -from nova import test - -class KeeperTestCase(test.TrialTestCase): - """ - Basic persistence tests for Keeper datastore. - Generalize, then use these to support - migration to redis / cassandra / multiple stores. - """ - - def setUp(self): - super(KeeperTestCase, self).setUp() - self.keeper = datastore.Keeper('test') - - def tearDown(self): - super(KeeperTestCase, self).tearDown() - self.keeper.clear() - - def test_store_strings(self): - """ - Confirm that simple strings go in and come out safely. - Should also test unicode strings. - """ - randomstring = ''.join( - [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') - for _x in xrange(20)] - ) - self.keeper['test_string'] = randomstring - self.assertEqual(randomstring, self.keeper['test_string']) - - def test_store_dicts(self): - """ - Arbitrary dictionaries should be storable. - """ - test_dict = {'key_one': 'value_one'} - self.keeper['test_dict'] = test_dict - self.assertEqual(test_dict['key_one'], - self.keeper['test_dict']['key_one']) - - def test_sets(self): - """ - A keeper dict should be self-serializing. - """ - self.keeper.set_add('test_set', 'foo') - test_dict = {'arbitrary': 'dict of stuff'} - self.keeper.set_add('test_set', test_dict) - self.assertTrue(self.keeper.set_is_member('test_set', 'foo')) - self.assertFalse(self.keeper.set_is_member('test_set', 'bar')) - self.keeper.set_remove('test_set', 'foo') - self.assertFalse(self.keeper.set_is_member('test_set', 'foo')) - rv = self.keeper.set_fetch('test_set') - self.assertEqual(test_dict, rv.next()) - self.keeper.set_remove('test_set', test_dict) -