x/gantt
Code Issues Proposed changes

removed all references to keeper

Browse Source
This commit is contained in:
Vishvananda Ishaya
2010-06-29 18:02:00 -05:00
parent 9f1977de0f
commit c5e33bc515
6 changed files with 3 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nova/auth/users.py
View File

@@ -239,7 +239,6 @@ class Project(Group):
def __init__(self, id, project_manager_id, description, member_ids):
self.project_manager_id = project_manager_id
super(Project, self).__init__(id, description, member_ids)
self.keeper = datastore.Keeper(prefix="project-")
@property
def project_manager(self):

257
nova/datastore.py
View File

@@ -21,19 +21,10 @@
"""
Datastore:
Providers the Keeper class, a simple pseudo-dictionary that
persists on disk.
MAKE Sure that ReDIS is running, and your flags are set properly,
before trying to run this.
"""
import json
import logging
import os
import sqlite3
import time
from nova import vendor
import redis
@@ -42,15 +33,11 @@ from nova import utils
FLAGS = flags.FLAGS
flags.DEFINE_string('datastore_path', utils.abspath('../keeper'),
'where keys are stored on disk')
flags.DEFINE_string('redis_host', '127.0.0.1',
'Host that redis is running on.')
flags.DEFINE_integer('redis_port', 6379,
'Port that redis is running on.')
flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away')
flags.DEFINE_string('keeper_backend', 'redis',
'which backend to use for keeper')
class Redis(object):
@@ -61,247 +48,9 @@ class Redis(object):
@classmethod
def instance(cls):
if not hasattr(cls, '_instance'):
inst = redis.Redis(host=FLAGS.redis_host, port=FLAGS.redis_port, db=FLAGS.redis_db)
inst = redis.Redis(host=FLAGS.redis_host,
port=FLAGS.redis_port,
db=FLAGS.redis_db)
cls._instance = inst
return cls._instance
def slugify(key, prefix=None):
"""
Key has to be a valid filename. Slugify solves that.
"""
return "%s%s" % (prefix, key)
class SqliteKeeper(object):
""" Keeper implementation in SQLite, mostly for in-memory testing """
_conn = {} # class variable
def __init__(self, prefix):
self.prefix = prefix
@property
def conn(self):
if self.prefix not in self.__class__._conn:
logging.debug('no sqlite connection (%s), making new', self.prefix)
if FLAGS.datastore_path != ':memory:':
try:
os.mkdir(FLAGS.datastore_path)
except Exception:
pass
conn = sqlite3.connect(os.path.join(
FLAGS.datastore_path, '%s.sqlite' % self.prefix))
else:
conn = sqlite3.connect(':memory:')
c = conn.cursor()
try:
c.execute('''CREATE TABLE data (item text, value text)''')
conn.commit()
except Exception:
logging.exception('create table failed')
finally:
c.close()
self.__class__._conn[self.prefix] = conn
return self.__class__._conn[self.prefix]
def __delitem__(self, item):
#logging.debug('sqlite deleting %s', item)
c = self.conn.cursor()
try:
c.execute('DELETE FROM data WHERE item = ?', (item, ))
self.conn.commit()
except Exception:
logging.exception('delete failed: %s', item)
finally:
c.close()
def __getitem__(self, item):
#logging.debug('sqlite getting %s', item)
result = None
c = self.conn.cursor()
try:
c.execute('SELECT value FROM data WHERE item = ?', (item, ))
row = c.fetchone()
if row:
result = json.loads(row[0])
else:
result = None
except Exception:
logging.exception('select failed: %s', item)
finally:
c.close()
#logging.debug('sqlite got %s: %s', item, result)
return result
def __setitem__(self, item, value):
serialized_value = json.dumps(value)
insert = True
if self[item] is not None:
insert = False
#logging.debug('sqlite insert %s: %s', item, value)
c = self.conn.cursor()
try:
if insert:
c.execute('INSERT INTO data VALUES (?, ?)',
(item, serialized_value))
else:
c.execute('UPDATE data SET item=?, value=? WHERE item = ?',
(item, serialized_value, item))
self.conn.commit()
except Exception:
logging.exception('select failed: %s', item)
finally:
c.close()
def clear(self):
if self.prefix not in self.__class__._conn:
return
self.conn.close()
if FLAGS.datastore_path != ':memory:':
os.unlink(os.path.join(FLAGS.datastore_path, '%s.sqlite' % self.prefix))
del self.__class__._conn[self.prefix]
def clear_all(self):
for k, conn in self.__class__._conn.iteritems():
conn.close()
if FLAGS.datastore_path != ':memory:':
os.unlink(os.path.join(FLAGS.datastore_path,
'%s.sqlite' % self.prefix))
self.__class__._conn = {}
def set_add(self, item, value):
group = self[item]
if not group:
group = []
group.append(value)
self[item] = group
def set_is_member(self, item, value):
group = self[item]
if not group:
return False
return value in group
def set_remove(self, item, value):
group = self[item]
if not group:
group = []
group.remove(value)
self[item] = group
def set_members(self, item):
group = self[item]
if not group:
group = []
return group
def set_fetch(self, item):
# TODO(termie): I don't really know what set_fetch is supposed to do
group = self[item]
if not group:
group = []
return iter(group)
class JsonKeeper(object):
"""
Simple dictionary class that persists using
JSON in files saved to disk.
"""
def __init__(self, prefix):
self.prefix = prefix
def __delitem__(self, item):
"""
Removing a key means deleting a file from disk.
"""
item = slugify(item, self.prefix)
path = "%s/%s" % (FLAGS.datastore_path, item)
if os.path.isfile(path):
os.remove(path)
def __getitem__(self, item):
"""
Fetch file contents and dejsonify them.
"""
item = slugify(item, self.prefix)
path = "%s/%s" % (FLAGS.datastore_path, item)
if os.path.isfile(path):
return json.load(open(path, 'r'))
return None
def __setitem__(self, item, value):
"""
JSON encode value and save to file.
"""
item = slugify(item, self.prefix)
path = "%s/%s" % (FLAGS.datastore_path, item)
with open(path, "w") as blobfile:
blobfile.write(json.dumps(value))
return value
class RedisKeeper(object):
"""
Simple dictionary class that persists using
ReDIS.
"""
def __init__(self, prefix="redis-"):
self.prefix = prefix
#Redis.instance().ping()
def __setitem__(self, item, value):
"""
JSON encode value and save to file.
"""
item = slugify(item, self.prefix)
Redis.instance().set(item, json.dumps(value))
return value
def __getitem__(self, item):
item = slugify(item, self.prefix)
value = Redis.instance().get(item)
if value:
return json.loads(value)
def __delitem__(self, item):
item = slugify(item, self.prefix)
return Redis.instance().delete(item)
def clear(self):
raise NotImplementedError()
def clear_all(self):
raise NotImplementedError()
def set_add(self, item, value):
item = slugify(item, self.prefix)
return Redis.instance().sadd(item, json.dumps(value))
def set_is_member(self, item, value):
item = slugify(item, self.prefix)
return Redis.instance().sismember(item, json.dumps(value))
def set_remove(self, item, value):
item = slugify(item, self.prefix)
return Redis.instance().srem(item, json.dumps(value))
def set_members(self, item):
item = slugify(item, self.prefix)
return [json.loads(v) for v in Redis.instance().smembers(item)]
def set_fetch(self, item):
item = slugify(item, self.prefix)
for obj in Redis.instance().sinter([item]):
yield json.loads(obj)
def Keeper(prefix=''):
KEEPERS = {'redis': RedisKeeper,
'sqlite': SqliteKeeper}
return KEEPERS[FLAGS.keeper_backend](prefix)

5
nova/test.py
View File

@@ -36,7 +36,6 @@ from twisted.python import failure
from twisted.trial import unittest as trial_unittest
import stubout
from nova import datastore
from nova import fakerabbit
from nova import flags
@@ -79,10 +78,6 @@ class TrialTestCase(trial_unittest.TestCase):
if FLAGS.fake_rabbit:
fakerabbit.reset_all()
# attempt to wipe all keepers
#keeper = datastore.Keeper()
#keeper.clear_all()
def flags(self, **kw):
for k, v in kw.iteritems():
if k in self.flag_overrides:

80
nova/tests/datastore_unittest.py
View File

@@ -1,80 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 Anso Labs, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nova import test
from nova import datastore
import random
class KeeperTestCase(test.BaseTestCase):
"""
Basic persistence tests for Keeper datastore.
Generalize, then use these to support
migration to redis / cassandra / multiple stores.
"""
def __init__(self, *args, **kwargs):
"""
Create a new keeper instance for test keys.
"""
super(KeeperTestCase, self).__init__(*args, **kwargs)
self.keeper = datastore.Keeper('test-')
def tear_down(self):
"""
Scrub out test keeper data.
"""
pass
def test_store_strings(self):
"""
Confirm that simple strings go in and come out safely.
Should also test unicode strings.
"""
randomstring = ''.join(
[random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-')
for _x in xrange(20)]
)
self.keeper['test_string'] = randomstring
self.assertEqual(randomstring, self.keeper['test_string'])
def test_store_dicts(self):
"""
Arbitrary dictionaries should be storable.
"""
test_dict = {'key_one': 'value_one'}
self.keeper['test_dict'] = test_dict
self.assertEqual(test_dict['key_one'],
self.keeper['test_dict']['key_one'])
def test_sets(self):
"""
A keeper dict should be self-serializing.
"""
self.keeper.set_add('test_set', 'foo')
test_dict = {'arbitrary': 'dict of stuff'}
self.keeper.set_add('test_set', test_dict)
self.assertTrue(self.keeper.set_is_member('test_set', 'foo'))
self.assertFalse(self.keeper.set_is_member('test_set', 'bar'))
self.keeper.set_remove('test_set', 'foo')
self.assertFalse(self.keeper.set_is_member('test_set', 'foo'))
rv = self.keeper.set_fetch('test_set')
self.assertEqual(test_dict, rv.next())
self.keeper.set_remove('test_set', test_dict)

2
nova/tests/fake_flags.py
View File

@@ -27,6 +27,4 @@ FLAGS.fake_storage = True
FLAGS.fake_rabbit = True
FLAGS.fake_network = True
FLAGS.fake_users = True
#FLAGS.keeper_backend = 'sqlite'
FLAGS.datastore_path = ':memory:'
FLAGS.verbose = True

76
nova/tests/keeper_unittest.py
View File

@@ -1,76 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 Anso Labs, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
from nova import datastore
from nova import test
class KeeperTestCase(test.TrialTestCase):
"""
Basic persistence tests for Keeper datastore.
Generalize, then use these to support
migration to redis / cassandra / multiple stores.
"""
def setUp(self):
super(KeeperTestCase, self).setUp()
self.keeper = datastore.Keeper('test')
def tearDown(self):
super(KeeperTestCase, self).tearDown()
self.keeper.clear()
def test_store_strings(self):
"""
Confirm that simple strings go in and come out safely.
Should also test unicode strings.
"""
randomstring = ''.join(
[random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-')
for _x in xrange(20)]
)
self.keeper['test_string'] = randomstring
self.assertEqual(randomstring, self.keeper['test_string'])
def test_store_dicts(self):
"""
Arbitrary dictionaries should be storable.
"""
test_dict = {'key_one': 'value_one'}
self.keeper['test_dict'] = test_dict
self.assertEqual(test_dict['key_one'],
self.keeper['test_dict']['key_one'])
def test_sets(self):
"""
A keeper dict should be self-serializing.
"""
self.keeper.set_add('test_set', 'foo')
test_dict = {'arbitrary': 'dict of stuff'}
self.keeper.set_add('test_set', test_dict)
self.assertTrue(self.keeper.set_is_member('test_set', 'foo'))
self.assertFalse(self.keeper.set_is_member('test_set', 'bar'))
self.keeper.set_remove('test_set', 'foo')
self.assertFalse(self.keeper.set_is_member('test_set', 'foo'))
rv = self.keeper.set_fetch('test_set')
self.assertEqual(test_dict, rv.next())
self.keeper.set_remove('test_set', test_dict)