237 lines
10 KiB
Python
237 lines
10 KiB
Python
"""\
|
|
@file db_pool.py
|
|
@brief Uses saranwrap to implement a pool of nonblocking database connections to a db server.
|
|
|
|
Copyright (c) 2007, Linden Research, Inc.
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in
|
|
all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
THE SOFTWARE.
|
|
"""
|
|
|
|
import os, sys
|
|
|
|
from eventlet.pools import Pool
|
|
from eventlet.processes import DeadProcess
|
|
from eventlet import saranwrap
|
|
|
|
class DatabaseConnector(object):
|
|
"""\
|
|
@brief This is an object which will maintain a collection of database
|
|
connection pools keyed on host,databasename"""
|
|
def __init__(self, module, credentials, min_size = 0, max_size = 4, conn_pool=None, *args, **kwargs):
|
|
"""\
|
|
@brief constructor
|
|
@param min_size the minimum size of a child pool.
|
|
@param max_size the maximum size of a child pool."""
|
|
assert(module)
|
|
self._conn_pool_class = conn_pool
|
|
if self._conn_pool_class is None:
|
|
self._conn_pool_class = ConnectionPool
|
|
self._module = module
|
|
self._min_size = min_size
|
|
self._max_size = max_size
|
|
self._args = args
|
|
self._kwargs = kwargs
|
|
self._credentials = credentials # this is a map of hostname to username/password
|
|
self._databases = {}
|
|
|
|
def credentials_for(self, host):
|
|
if host in self._credentials:
|
|
return self._credentials[host]
|
|
else:
|
|
return self._credentials.get('default', None)
|
|
|
|
def get(self, host, dbname):
|
|
key = (host, dbname)
|
|
if key not in self._databases:
|
|
new_kwargs = self._kwargs.copy()
|
|
new_kwargs['db'] = dbname
|
|
new_kwargs['host'] = host
|
|
new_kwargs.update(self.credentials_for(host))
|
|
dbpool = self._conn_pool_class(self._module, min_size=self._min_size, max_size=self._max_size,
|
|
*self._args, **new_kwargs)
|
|
self._databases[key] = dbpool
|
|
|
|
return self._databases[key]
|
|
|
|
class BaseConnectionPool(Pool):
|
|
# *TODO: we need to expire and close connections if they've been
|
|
# idle for a while, so that system-wide connection count doesn't
|
|
# monotonically increase forever
|
|
def __init__(self, db_module, min_size = 0, max_size = 4, *args, **kwargs):
|
|
assert(db_module)
|
|
self._db_module = db_module
|
|
self._args = args
|
|
self._kwargs = kwargs
|
|
super(BaseConnectionPool, self).__init__(min_size, max_size)
|
|
|
|
def get(self):
|
|
# wrap the connection for easier use
|
|
conn = super(BaseConnectionPool, self).get()
|
|
return PooledConnectionWrapper(conn, self)
|
|
|
|
def put(self, conn):
|
|
# rollback any uncommitted changes, so that the next client
|
|
# has a clean slate. This also pokes the connection to see if
|
|
# it's dead or None
|
|
try:
|
|
conn.rollback()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except AttributeError, e:
|
|
# this means it's already been destroyed, so we don't need to print anything
|
|
conn = None
|
|
except:
|
|
# we don't care what the exception was, we just know the
|
|
# connection is dead
|
|
print "WARNING: connection.rollback raised: %s" % (sys.exc_info()[1])
|
|
conn = None
|
|
|
|
# unwrap the connection for storage
|
|
if isinstance(conn, GenericConnectionWrapper):
|
|
if conn:
|
|
base = conn._base
|
|
conn._destroy()
|
|
conn = base
|
|
else:
|
|
conn = None
|
|
|
|
if conn is not None:
|
|
super(BaseConnectionPool, self).put(conn)
|
|
else:
|
|
self.current_size -= 1
|
|
|
|
def clear(self):
|
|
""" Close all connections that this pool still holds a reference to, leaving it empty."""
|
|
for conn in self.free_items:
|
|
try:
|
|
conn.close()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
pass # even if stuff happens here, we still want to at least try to close all the other connections
|
|
self.free_items.clear()
|
|
|
|
def __del__(self):
|
|
self.clear()
|
|
|
|
|
|
class SaranwrappedConnectionPool(BaseConnectionPool):
|
|
"""A pool which gives out saranwrapped database connections from a pool
|
|
"""
|
|
def create(self):
|
|
return saranwrap.wrap(self._db_module).connect(*self._args, **self._kwargs)
|
|
|
|
class TpooledConnectionPool(BaseConnectionPool):
|
|
"""A pool which gives out tpool.Proxy-based database connections from a pool.
|
|
"""
|
|
def create(self):
|
|
from eventlet import tpool
|
|
try:
|
|
# *FIX: this is a huge hack that will probably only work for MySQLdb
|
|
autowrap = (self._db_module.cursors.DictCursor,)
|
|
except:
|
|
autowrap = ()
|
|
return tpool.Proxy(self._db_module.connect(*self._args, **self._kwargs),
|
|
autowrap=autowrap)
|
|
|
|
class RawConnectionPool(BaseConnectionPool):
|
|
"""A pool which gives out plain database connections from a pool.
|
|
"""
|
|
def create(self):
|
|
return self._db_module.connect(*self._args, **self._kwargs)
|
|
|
|
# default connection pool is the tpool one
|
|
ConnectionPool = TpooledConnectionPool
|
|
|
|
|
|
class GenericConnectionWrapper(object):
|
|
def __init__(self, baseconn):
|
|
self._base = baseconn
|
|
def __enter__(self): return self._base.__enter__()
|
|
def __exit__(self, exc, value, tb): return self._base.__exit__(exc, value, tb)
|
|
def __repr__(self): return self._base.__repr__()
|
|
def affected_rows(self): return self._base.affected_rows()
|
|
def autocommit(self,*args, **kwargs): return self._base.autocommit(*args, **kwargs)
|
|
def begin(self): return self._base.begin()
|
|
def change_user(self,*args, **kwargs): return self._base.change_user(*args, **kwargs)
|
|
def character_set_name(self,*args, **kwargs): return self._base.character_set_name(*args, **kwargs)
|
|
def close(self,*args, **kwargs): return self._base.close(*args, **kwargs)
|
|
def commit(self,*args, **kwargs): return self._base.commit(*args, **kwargs)
|
|
def cursor(self, cursorclass=None, **kwargs): return self._base.cursor(cursorclass, **kwargs)
|
|
def dump_debug_info(self,*args, **kwargs): return self._base.dump_debug_info(*args, **kwargs)
|
|
def errno(self,*args, **kwargs): return self._base.errno(*args, **kwargs)
|
|
def error(self,*args, **kwargs): return self._base.error(*args, **kwargs)
|
|
def errorhandler(self, conn, curs, errcls, errval): return self._base.errorhandler(conn, curs, errcls, errval)
|
|
def literal(self, o): return self._base.literal(o)
|
|
def set_character_set(self, charset): return self._base.set_character_set(charset)
|
|
def set_sql_mode(self, sql_mode): return self._base.set_sql_mode(sql_mode)
|
|
def show_warnings(self): return self._base.show_warnings()
|
|
def warning_count(self): return self._base.warning_count()
|
|
def literal(self, o): return self._base.literal(o)
|
|
def ping(self,*args, **kwargs): return self._base.ping(*args, **kwargs)
|
|
def query(self,*args, **kwargs): return self._base.query(*args, **kwargs)
|
|
def rollback(self,*args, **kwargs): return self._base.rollback(*args, **kwargs)
|
|
def select_db(self,*args, **kwargs): return self._base.select_db(*args, **kwargs)
|
|
def set_server_option(self,*args, **kwargs): return self._base.set_server_option(*args, **kwargs)
|
|
def set_character_set(self, charset): return self._base.set_character_set(charset)
|
|
def set_sql_mode(self, sql_mode): return self._base.set_sql_mode(sql_mode)
|
|
def server_capabilities(self,*args, **kwargs): return self._base.server_capabilities(*args, **kwargs)
|
|
def show_warnings(self): return self._base.show_warnings()
|
|
def shutdown(self,*args, **kwargs): return self._base.shutdown(*args, **kwargs)
|
|
def sqlstate(self,*args, **kwargs): return self._base.sqlstate(*args, **kwargs)
|
|
def stat(self,*args, **kwargs): return self._base.stat(*args, **kwargs)
|
|
def store_result(self,*args, **kwargs): return self._base.store_result(*args, **kwargs)
|
|
def string_literal(self,*args, **kwargs): return self._base.string_literal(*args, **kwargs)
|
|
def thread_id(self,*args, **kwargs): return self._base.thread_id(*args, **kwargs)
|
|
def use_result(self,*args, **kwargs): return self._base.use_result(*args, **kwargs)
|
|
def warning_count(self): return self._base.warning_count()
|
|
|
|
|
|
class PooledConnectionWrapper(GenericConnectionWrapper):
|
|
""" A connection wrapper where:
|
|
- the close method returns the connection to the pool instead of closing it directly
|
|
- you can do if conn:
|
|
- returns itself to the pool if it gets garbage collected
|
|
"""
|
|
def __init__(self, baseconn, pool):
|
|
super(PooledConnectionWrapper, self).__init__(baseconn)
|
|
self._pool = pool
|
|
|
|
def __nonzero__(self):
|
|
return (hasattr(self, '_base') and bool(self._base))
|
|
|
|
def _destroy(self):
|
|
self._pool = None
|
|
try:
|
|
del self._base
|
|
except AttributeError:
|
|
pass
|
|
|
|
def close(self):
|
|
""" Return the connection to the pool, and remove the
|
|
reference to it so that you can't use it again through this
|
|
wrapper object.
|
|
"""
|
|
if self and self._pool:
|
|
self._pool.put(self)
|
|
self._destroy()
|
|
|
|
def __del__(self):
|
|
self.close()
|