openstack/swift
Code Issues Proposed changes

Remove :memory: from DatabaseBrokers and unittests

Browse Source 
The SQLite in-memory databases have been great for testing but
as the swift DatabaseBroker's have become more complex, the limitations
of in memory databases are being reached. Mostly due
to the introduction of container sharding where a broker sometimes needs
to make multiple connections to the same database as the same time.

Rather then rework the real broker logic to better support in-memory
testing, it's actually easier to just remove the in-memory broker tests
and use a "real" broker in a tempdir. This allows us to better test how
brokers behave in real life, pending files and all.

This patch replaces all the :memory: brokers in the tests with real ones
placed in a tempdir. To achieve this, we new base unittest class `TestDBBase`
has been added that creates, cleans up and provides some helper methods
to manage the db path and location.

Further, all references to :memory: in the Database brokers have been
removed.

Change-Id: I5983132f776b84db634fef39c833d5cfdce11980
This commit is contained in:
Matthew Oliver 2022-07-07 17:27:49 +10:00
parent 475cdba65b
commit a548da916f
8 changed files with 345 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
swift/common/db.py
View File

@ -205,7 +205,7 @@ def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
factory=GreenDBConnection, timeout=timeout) factory=GreenDBConnection, timeout=timeout)
if QUERY_LOGGING and logger and not six.PY2: if QUERY_LOGGING and logger and not six.PY2:
conn.set_trace_callback(logger.debug) conn.set_trace_callback(logger.debug)
if path != ':memory:' and not okay_to_create: if not okay_to_create:
# attempt to detect and fail when connect creates the db file # attempt to detect and fail when connect creates the db file
stat = os.stat(path) stat = os.stat(path)
if stat.st_size == 0 and stat.st_ctime >= connect_time: if stat.st_size == 0 and stat.st_ctime >= connect_time:
@ -359,17 +359,13 @@ class DatabaseBroker(object):
:param put_timestamp: internalized timestamp of initial PUT request :param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers :param storage_policy_index: only required for containers
""" """
if self._db_file == ':memory:': mkdirs(self.db_dir)
tmp_db_file = None fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
conn = get_db_connection(self._db_file, self.timeout, self.logger) os.close(fd)
else: conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
mkdirs(self.db_dir) factory=GreenDBConnection, timeout=0)
fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir) if QUERY_LOGGING and not six.PY2:
os.close(fd) conn.set_trace_callback(self.logger.debug)
conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
factory=GreenDBConnection, timeout=0)
if QUERY_LOGGING and not six.PY2:
conn.set_trace_callback(self.logger.debug)
# creating dbs implicitly does a lot of transactions, so we # creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end. # pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur: with closing(conn.cursor()) as cur:
@ -541,7 +537,7 @@ class DatabaseBroker(object):
def get(self): def get(self):
"""Use with the "with" statement; returns a database connection.""" """Use with the "with" statement; returns a database connection."""
if not self.conn: if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file): if os.path.exists(self.db_file):
try: try:
self.conn = get_db_connection(self.db_file, self.timeout, self.conn = get_db_connection(self.db_file, self.timeout,
self.logger) self.logger)
@ -569,7 +565,7 @@ class DatabaseBroker(object):
def lock(self): def lock(self):
"""Use with the "with" statement; locks a database.""" """Use with the "with" statement; locks a database."""
if not self.conn: if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file): if os.path.exists(self.db_file):
self.conn = get_db_connection(self.db_file, self.timeout, self.conn = get_db_connection(self.db_file, self.timeout,
self.logger) self.logger)
else: else:
@ -637,7 +633,7 @@ class DatabaseBroker(object):
:returns: True if the DB is considered to be deleted, False otherwise :returns: True if the DB is considered to be deleted, False otherwise
""" """
if self.db_file != ':memory:' and not os.path.exists(self.db_file): if not os.path.exists(self.db_file):
return True return True
self._commit_puts_stale_ok() self._commit_puts_stale_ok()
with self.get() as conn: with self.get() as conn:
@ -770,8 +766,8 @@ class DatabaseBroker(object):
""" """
Put a record into the DB. If the DB has an associated pending file with Put a record into the DB. If the DB has an associated pending file with
space then the record is appended to that file and a commit to the DB space then the record is appended to that file and a commit to the DB
is deferred. If the DB is in-memory or its pending file is full then is deferred. If its pending file is full then the record will be
the record will be committed immediately. committed immediately.
:param record: a record to be added to the DB. :param record: a record to be added to the DB.
:raises DatabaseConnectionError: if the DB file does not exist or if :raises DatabaseConnectionError: if the DB file does not exist or if
@ -779,9 +775,6 @@ class DatabaseBroker(object):
:raises LockTimeout: if a timeout occurs while waiting to take a lock :raises LockTimeout: if a timeout occurs while waiting to take a lock
to write to the pending file. to write to the pending file.
""" """
if self._db_file == ':memory:':
self.merge_items([record])
return
if not os.path.exists(self.db_file): if not os.path.exists(self.db_file):
raise DatabaseConnectionError(self.db_file, "DB doesn't exist") raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
if self.skip_commits: if self.skip_commits:
@ -807,8 +800,7 @@ class DatabaseBroker(object):
fp.flush() fp.flush()
def _skip_commit_puts(self): def _skip_commit_puts(self):
return (self._db_file == ':memory:' or self.skip_commits or not return self.skip_commits or not os.path.exists(self.pending_file)
os.path.exists(self.pending_file))
def _commit_puts(self, item_list=None): def _commit_puts(self, item_list=None):
""" """
@ -922,7 +914,7 @@ class DatabaseBroker(object):
within 512k of a boundary, it allocates to the next boundary. within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
""" """
if not DB_PREALLOCATION or self._db_file == ':memory:': if not DB_PREALLOCATION:
return return
MB = (1024 * 1024) MB = (1024 * 1024)

11
swift/container/backend.py
View File

@ -348,10 +348,7 @@ class ContainerBroker(DatabaseBroker):
stale_reads_ok=False, skip_commits=False, stale_reads_ok=False, skip_commits=False,
force_db_file=False): force_db_file=False):
self._init_db_file = db_file self._init_db_file = db_file
if db_file == ':memory:': base_db_file = make_db_file_path(db_file, None)
base_db_file = db_file
else:
base_db_file = make_db_file_path(db_file, None)
super(ContainerBroker, self).__init__( super(ContainerBroker, self).__init__(
base_db_file, timeout, logger, account, container, pending_timeout, base_db_file, timeout, logger, account, container, pending_timeout,
stale_reads_ok, skip_commits=skip_commits) stale_reads_ok, skip_commits=skip_commits)
@ -396,8 +393,6 @@ class ContainerBroker(DatabaseBroker):
""" """
Returns the current state of on disk db files. Returns the current state of on disk db files.
""" """
if self._db_file == ':memory:':
return UNSHARDED
if not self.db_files: if not self.db_files:
return NOTFOUND return NOTFOUND
if len(self.db_files) > 1: if len(self.db_files) > 1:
@ -441,8 +436,6 @@ class ContainerBroker(DatabaseBroker):
""" """
Reloads the cached list of valid on disk db files for this broker. Reloads the cached list of valid on disk db files for this broker.
""" """
if self._db_file == ':memory:':
return
# reset connection so the next access will use the correct DB file # reset connection so the next access will use the correct DB file
self.conn = None self.conn = None
self._db_files = get_db_files(self._init_db_file) self._db_files = get_db_files(self._init_db_file)
@ -849,7 +842,7 @@ class ContainerBroker(DatabaseBroker):
:returns: a tuple, in the form (info, is_deleted) info is a dict as :returns: a tuple, in the form (info, is_deleted) info is a dict as
returned by get_info and is_deleted is a boolean. returned by get_info and is_deleted is a boolean.
""" """
if self.db_file != ':memory:' and not os.path.exists(self.db_file): if not os.path.exists(self.db_file):
return {}, True return {}, True
info = self.get_info() info = self.get_info()
return info, self._is_deleted_info(**info) return info, self._is_deleted_info(**info)

7
test/unit/__init__.py
View File

@ -42,6 +42,7 @@ import random
import errno import errno
import xattr import xattr
from io import BytesIO from io import BytesIO
from uuid import uuid4
import six import six
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
@ -1408,3 +1409,9 @@ def group_by_byte(contents):
return [ return [
(char, sum(1 for _ in grp)) (char, sum(1 for _ in grp))
for char, grp in itertools.groupby(byte_iter)] for char, grp in itertools.groupby(byte_iter)]
def generate_db_path(tempdir, server_type):
return os.path.join(
tempdir, '%ss' % server_type, 'part', 'suffix', 'hash',
'%s-%s.db' % (server_type, uuid4()))

7
test/unit/account/test_auditor.py
View File

@ -53,14 +53,15 @@ class TestAuditorRealBrokerMigration(
self.assertUnmigrated(self.broker) self.assertUnmigrated(self.broker)
# run auditor, and validate migration # run auditor, and validate migration
conf = {'devices': self.tempdir, 'mount_check': False, conf = {'devices': self.testdir, 'mount_check': False,
'recon_cache_path': self.tempdir} 'recon_cache_path': self.testdir}
test_auditor = auditor.AccountAuditor(conf, logger=debug_logger()) test_auditor = auditor.AccountAuditor(conf, logger=debug_logger())
test_auditor.run_once() test_auditor.run_once()
self.restore_account_broker() self.restore_account_broker()
broker = auditor.AccountBroker(self.db_path) broker = auditor.AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal, 0)
# go after rows directly to avoid unintentional migration # go after rows directly to avoid unintentional migration
with broker.get() as conn: with broker.get() as conn:
rows = conn.execute(''' rows = conn.execute('''

109
test/unit/account/test_backend.py
View File

@ -17,13 +17,10 @@
from collections import defaultdict from collections import defaultdict
import json import json
import unittest
import pickle import pickle
import os import os
from time import sleep, time from time import sleep, time
from uuid import uuid4 from uuid import uuid4
from tempfile import mkdtemp
from shutil import rmtree
import sqlite3 import sqlite3
import itertools import itertools
from contextlib import contextmanager from contextlib import contextmanager
@ -46,23 +43,25 @@ from test.unit.common import test_db
@patch_policies @patch_policies
class TestAccountBroker(unittest.TestCase): class TestAccountBroker(test_db.TestDbBase):
"""Tests for AccountBroker""" """Tests for AccountBroker"""
def setUp(self): def setUp(self):
super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now" # tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1) self.ts = make_timestamp_iter(offset=-1)
def test_creation(self): def test_creation(self):
# Test AccountBroker.__init__ # Test AccountBroker.__init__
broker = AccountBroker(':memory:', account='a') db_file = self.get_db_path()
self.assertEqual(broker.db_file, ':memory:') broker = AccountBroker(db_file, account='a')
self.assertEqual(broker.db_file, db_file)
try: try:
with broker.get() as conn: with broker.get() as conn:
pass pass
except DatabaseConnectionError as e: except DatabaseConnectionError as e:
self.assertTrue(hasattr(e, 'path')) self.assertTrue(hasattr(e, 'path'))
self.assertEqual(e.path, ':memory:') self.assertEqual(e.path, db_file)
self.assertTrue(hasattr(e, 'msg')) self.assertTrue(hasattr(e, 'msg'))
self.assertEqual(e.msg, "DB doesn't exist") self.assertEqual(e.msg, "DB doesn't exist")
except Exception as e: except Exception as e:
@ -76,7 +75,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1) self.assertEqual(curs.fetchall()[0][0], 1)
def test_initialize_fail(self): def test_initialize_fail(self):
broker = AccountBroker(':memory:') broker = AccountBroker(self.get_db_path())
with self.assertRaises(ValueError) as cm: with self.assertRaises(ValueError) as cm:
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
self.assertEqual(str(cm.exception), 'Attempting to create a new' self.assertEqual(str(cm.exception), 'Attempting to create a new'
@ -85,7 +84,7 @@ class TestAccountBroker(unittest.TestCase):
def test_exception(self): def test_exception(self):
# Test AccountBroker throwing a conn away after exception # Test AccountBroker throwing a conn away after exception
first_conn = None first_conn = None
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
with broker.get() as conn: with broker.get() as conn:
first_conn = conn first_conn = conn
@ -99,7 +98,7 @@ class TestAccountBroker(unittest.TestCase):
def test_empty(self): def test_empty(self):
# Test AccountBroker.empty # Test AccountBroker.empty
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
self.assertTrue(broker.empty()) self.assertTrue(broker.empty())
broker.put_container('o', Timestamp.now().internal, 0, 0, 0, broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
@ -112,12 +111,12 @@ class TestAccountBroker(unittest.TestCase):
def test_is_status_deleted(self): def test_is_status_deleted(self):
# Test AccountBroker.is_status_deleted # Test AccountBroker.is_status_deleted
broker1 = AccountBroker(':memory:', account='a') broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp.now().internal) broker1.initialize(Timestamp.now().internal)
self.assertFalse(broker1.is_status_deleted()) self.assertFalse(broker1.is_status_deleted())
broker1.delete_db(Timestamp.now().internal) broker1.delete_db(Timestamp.now().internal)
self.assertTrue(broker1.is_status_deleted()) self.assertTrue(broker1.is_status_deleted())
broker2 = AccountBroker(':memory:', account='a') broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp.now().internal) broker2.initialize(Timestamp.now().internal)
# Set delete_timestamp greater than put_timestamp # Set delete_timestamp greater than put_timestamp
broker2.merge_timestamps( broker2.merge_timestamps(
@ -126,10 +125,12 @@ class TestAccountBroker(unittest.TestCase):
self.assertTrue(broker2.is_status_deleted()) self.assertTrue(broker2.is_status_deleted())
def test_reclaim(self): def test_reclaim(self):
broker = AccountBroker(':memory:', account='test_account') broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
broker.put_container('c', Timestamp.now().internal, 0, 0, 0, broker.put_container('c', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM container " "SELECT count(*) FROM container "
@ -148,6 +149,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001) sleep(.00001)
broker.put_container('c', 0, Timestamp.now().internal, 0, 0, broker.put_container('c', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM container " "SELECT count(*) FROM container "
@ -176,6 +178,7 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
broker._commit_puts()
broker.reclaim(Timestamp.now().internal, time()) broker.reclaim(Timestamp.now().internal, time())
# Now delete the account # Now delete the account
broker.delete_db(Timestamp.now().internal) broker.delete_db(Timestamp.now().internal)
@ -193,7 +196,7 @@ class TestAccountBroker(unittest.TestCase):
random.seed(now) random.seed(now)
random.shuffle(container_specs) random.shuffle(container_specs)
policy_indexes = list(p.idx for p in POLICIES) policy_indexes = list(p.idx for p in POLICIES)
broker = AccountBroker(':memory:', account='test_account') broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
for i, container_spec in enumerate(container_specs): for i, container_spec in enumerate(container_specs):
# with container12 before container2 and shuffled ts.internal we # with container12 before container2 and shuffled ts.internal we
@ -206,6 +209,9 @@ class TestAccountBroker(unittest.TestCase):
else: else:
broker.put_container(name, ts.internal, 0, 0, 0, pidx) broker.put_container(name, ts.internal, 0, 0, 0, pidx)
# commit pending file into db
broker._commit_puts()
def count_reclaimable(conn, reclaim_age): def count_reclaimable(conn, reclaim_age):
return conn.execute( return conn.execute(
"SELECT count(*) FROM container " "SELECT count(*) FROM container "
@ -251,7 +257,7 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_db_status(self): def test_delete_db_status(self):
start = next(self.ts) start = next(self.ts)
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(start.internal) broker.initialize(start.internal)
info = broker.get_info() info = broker.get_info()
self.assertEqual(info['put_timestamp'], start.internal) self.assertEqual(info['put_timestamp'], start.internal)
@ -273,10 +279,12 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_container(self): def test_delete_container(self):
# Test AccountBroker.delete_container # Test AccountBroker.delete_container
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
broker.put_container('o', Timestamp.now().internal, 0, 0, 0, broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM container " "SELECT count(*) FROM container "
@ -287,6 +295,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001) sleep(.00001)
broker.put_container('o', 0, Timestamp.now().internal, 0, 0, broker.put_container('o', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM container " "SELECT count(*) FROM container "
@ -297,13 +306,15 @@ class TestAccountBroker(unittest.TestCase):
def test_put_container(self): def test_put_container(self):
# Test AccountBroker.put_container # Test AccountBroker.put_container
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
# Create initial container # Create initial container
timestamp = Timestamp.now().internal timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -317,6 +328,7 @@ class TestAccountBroker(unittest.TestCase):
# Reput same event # Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -332,6 +344,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -346,6 +359,7 @@ class TestAccountBroker(unittest.TestCase):
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0, broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -360,6 +374,7 @@ class TestAccountBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0, broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -378,6 +393,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0, broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -393,6 +409,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0], "SELECT name FROM container").fetchone()[0],
@ -405,7 +422,7 @@ class TestAccountBroker(unittest.TestCase):
def test_get_info(self): def test_get_info(self):
# Test AccountBroker.get_info # Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1') broker = AccountBroker(self.get_db_path(), account='test1')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
info = broker.get_info() info = broker.get_info()
@ -452,7 +469,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_containers_iter(self): def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter # Test AccountBroker.list_containers_iter
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
for cont1 in range(4): for cont1 in range(4):
for cont2 in range(125): for cont2 in range(125):
@ -582,7 +599,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self): def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter # Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_container( broker.put_container(
@ -617,7 +634,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_container_iter_with_reserved_name(self): def test_list_container_iter_with_reserved_name(self):
# Test ContainerBroker.list_objects_iter # Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0) broker.initialize(next(self.ts).internal, 0)
broker.put_container( broker.put_container(
@ -754,11 +771,13 @@ class TestAccountBroker(unittest.TestCase):
} }
failures = [] failures = []
for expected in expectations: for expected in expectations:
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0) broker.initialize(next(self.ts).internal, 0)
for name in expected['containers']: for name in expected['containers']:
broker.put_container(name, next(self.ts).internal, 0, 0, 0, broker.put_container(name, next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
# commit pending file into db
broker._commit_puts()
params = default_listing_params.copy() params = default_listing_params.copy()
params.update(expected['params']) params.update(expected['params'])
listing = list(c[0] for c in broker.list_containers_iter(**params)) listing = list(c[0] for c in broker.list_containers_iter(**params))
@ -773,7 +792,7 @@ class TestAccountBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self): def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an # Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter # account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp.now().internal, 0, 0, 0, broker.put_container('a', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
@ -814,7 +833,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b']) self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_chexor(self): def test_chexor(self):
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal, broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0, Timestamp(0).internal, 0, 0,
@ -842,9 +861,9 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_info()['hash'], hashc) self.assertEqual(broker.get_info()['hash'], hashc)
def test_merge_items(self): def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a') broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal) broker1.initialize(Timestamp('1').internal)
broker2 = AccountBroker(':memory:', account='a') broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal) broker2.initialize(Timestamp('1').internal)
broker1.put_container('a', Timestamp(1).internal, 0, 0, 0, broker1.put_container('a', Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx) POLICIES.default.idx)
@ -869,10 +888,10 @@ class TestAccountBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}' snowman = u'\N{SNOWMAN}'
if six.PY2: if six.PY2:
snowman = snowman.encode('utf-8') snowman = snowman.encode('utf-8')
broker1 = AccountBroker(':memory:', account='a') broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
id1 = broker1.get_info()['id'] id1 = broker1.get_info()['id']
broker2 = AccountBroker(':memory:', account='a') broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
broker1.put_container(snowman, Timestamp(2).internal, 0, 1, 100, broker1.put_container(snowman, Timestamp(2).internal, 0, 1, 100,
POLICIES.default.idx) POLICIES.default.idx)
@ -974,7 +993,7 @@ class TestAccountBroker(unittest.TestCase):
StoragePolicy(2, 'two', False), StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)]) StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self): def test_get_policy_stats(self):
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal) broker.initialize(next(self.ts).internal)
# check empty policy_stats # check empty policy_stats
self.assertTrue(broker.empty()) self.assertTrue(broker.empty())
@ -1041,7 +1060,7 @@ class TestAccountBroker(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', False), @patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)]) StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self): def test_policy_stats_tracking(self):
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal) broker.initialize(next(self.ts).internal)
# policy 0 # policy 0
@ -1162,6 +1181,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp):
class TestCommonAccountBroker(test_db.TestExampleBroker): class TestCommonAccountBroker(test_db.TestExampleBroker):
broker_class = AccountBroker broker_class = AccountBroker
server_type = 'account'
def setUp(self): def setUp(self):
super(TestCommonAccountBroker, self).setUp() super(TestCommonAccountBroker, self).setUp()
@ -1183,13 +1203,14 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
""" """
def setUp(self): def setUp(self):
super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now" # tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1) self.ts = make_timestamp_iter(offset=-1)
self._imported_create_account_stat_table = \ self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \ AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table premetadata_create_account_stat_table
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
exc = None exc = None
with broker.get() as conn: with broker.get() as conn:
@ -1202,10 +1223,11 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
def tearDown(self): def tearDown(self):
AccountBroker.create_account_stat_table = \ AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table self._imported_create_account_stat_table
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat') conn.execute('SELECT metadata FROM account_stat')
super(TestAccountBrokerBeforeMetadata, self).tearDown()
def prespi_create_container_table(self, conn): def prespi_create_container_table(self, conn):
@ -1270,6 +1292,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
""" """
def setUp(self): def setUp(self):
super(TestAccountBrokerBeforeSPI, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now" # tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1) self.ts = make_timestamp_iter(offset=-1)
self._imported_create_container_table = \ self._imported_create_container_table = \
@ -1278,7 +1301,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
prespi_create_container_table prespi_create_container_table
self._imported_initialize = AccountBroker._initialize self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize AccountBroker._initialize = prespi_AccountBroker_initialize
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
exc = None exc = None
with broker.get() as conn: with broker.get() as conn:
@ -1299,10 +1322,11 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
AccountBroker.create_container_table = \ AccountBroker.create_container_table = \
self._imported_create_container_table self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize AccountBroker._initialize = self._imported_initialize
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container') conn.execute('SELECT storage_policy_index FROM container')
super(TestAccountBrokerBeforeSPI, self).tearDown()
@with_tempdir @with_tempdir
def test_policy_table_migration(self, tempdir): def test_policy_table_migration(self, tempdir):
@ -1681,7 +1705,7 @@ def pre_track_containers_create_container_table(self, conn):
""" + OLD_POLICY_STAT_TRIGGER_SCRIPT) """ + OLD_POLICY_STAT_TRIGGER_SCRIPT)
class AccountBrokerPreTrackContainerCountSetup(object): class AccountBrokerPreTrackContainerCountSetup(test_db.TestDbBase):
def assertUnmigrated(self, broker): def assertUnmigrated(self, broker):
with broker.get() as conn: with broker.get() as conn:
try: try:
@ -1696,6 +1720,7 @@ class AccountBrokerPreTrackContainerCountSetup(object):
'trying to select container_count from policy_stat!') 'trying to select container_count from policy_stat!')
def setUp(self): def setUp(self):
super(AccountBrokerPreTrackContainerCountSetup, self).setUp()
# use old version of policy_stat # use old version of policy_stat
self._imported_create_policy_stat_table = \ self._imported_create_policy_stat_table = \
AccountBroker.create_policy_stat_table AccountBroker.create_policy_stat_table
@ -1708,17 +1733,16 @@ class AccountBrokerPreTrackContainerCountSetup(object):
AccountBroker.create_container_table = \ AccountBroker.create_container_table = \
pre_track_containers_create_container_table pre_track_containers_create_container_table
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
self.assertUnmigrated(broker) self.assertUnmigrated(broker)
self.tempdir = mkdtemp()
# tests seem to assume x-timestamp was set by the proxy before "now" # tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1) self.ts = make_timestamp_iter(offset=-1)
self.db_path = os.path.join(self.tempdir, 'sda', 'accounts', self.db_path = os.path.join(self.testdir, 'sda', 'accounts',
'0', '0', '0', 'test.db') '0', '0', '0', 'test.db')
self.broker = AccountBroker(self.db_path, account='a') self.broker = AccountBroker(self.get_db_path(), account='a')
self.broker.initialize(next(self.ts).internal) self.broker.initialize(next(self.ts).internal)
# Common sanity-check that our starting, pre-migration state correctly # Common sanity-check that our starting, pre-migration state correctly
@ -1726,14 +1750,13 @@ class AccountBrokerPreTrackContainerCountSetup(object):
self.assertUnmigrated(self.broker) self.assertUnmigrated(self.broker)
def tearDown(self): def tearDown(self):
rmtree(self.tempdir, ignore_errors=True)
self.restore_account_broker() self.restore_account_broker()
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT container_count FROM policy_stat') conn.execute('SELECT container_count FROM policy_stat')
super(AccountBrokerPreTrackContainerCountSetup, self).tearDown()
def restore_account_broker(self): def restore_account_broker(self):
AccountBroker.create_policy_stat_table = \ AccountBroker.create_policy_stat_table = \
@ -1925,7 +1948,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
self.assertEqual(policy_stat['container_count'], 1) self.assertEqual(policy_stat['container_count'], 1)
def test_migrate_add_storage_policy_index_fail(self): def test_migrate_add_storage_policy_index_fail(self):
broker = AccountBroker(':memory:', account='a') broker = AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal) broker.initialize(Timestamp('1').internal)
with mock.patch.object( with mock.patch.object(
broker, 'create_policy_stat_table', broker, 'create_policy_stat_table',

13
test/unit/account/test_utils.py
View File

@ -26,6 +26,7 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name from swift.common.request_helpers import get_reserved_name
from test.unit import patch_policies, make_timestamp_iter from test.unit import patch_policies, make_timestamp_iter
from test.unit.common.test_db import TestDbBase
class TestFakeAccountBroker(unittest.TestCase): class TestFakeAccountBroker(unittest.TestCase):
@ -58,9 +59,11 @@ class TestFakeAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_policy_stats(), {}) self.assertEqual(broker.get_policy_stats(), {})
class TestAccountUtils(unittest.TestCase): class TestAccountUtils(TestDbBase):
server_type = 'account'
def setUp(self): def setUp(self):
super(TestAccountUtils, self).setUp()
self.ts = make_timestamp_iter() self.ts = make_timestamp_iter()
def test_get_response_headers_fake_broker(self): def test_get_response_headers_fake_broker(self):
@ -78,7 +81,7 @@ class TestAccountUtils(unittest.TestCase):
self.assertEqual(resp_headers, expected) self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self): def test_get_response_headers_empty_memory_broker(self):
broker = backend.AccountBroker(':memory:', account='a') broker = backend.AccountBroker(self.db_path, account='a')
now = time.time() now = time.time()
with mock.patch('time.time', new=lambda: now): with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal) broker.initialize(Timestamp(now).internal)
@ -94,7 +97,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies @patch_policies
def test_get_response_headers_with_data(self): def test_get_response_headers_with_data(self):
broker = backend.AccountBroker(':memory:', account='a') broker = backend.AccountBroker(self.db_path, account='a')
now = time.time() now = time.time()
with mock.patch('time.time', new=lambda: now): with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal) broker.initialize(Timestamp(now).internal)
@ -141,7 +144,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies @patch_policies
def test_get_response_headers_with_legacy_data(self): def test_get_response_headers_with_legacy_data(self):
broker = backend.AccountBroker(':memory:', account='a') broker = backend.AccountBroker(self.db_path, account='a')
now = time.time() now = time.time()
with mock.patch('time.time', new=lambda: now): with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal) broker.initialize(Timestamp(now).internal)
@ -213,7 +216,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', is_default=True)]) @patch_policies([StoragePolicy(0, 'zero', is_default=True)])
def test_account_listing_reserved_names(self): def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(':memory:', account='a') broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts) put_timestamp = next(self.ts)
now = time.time() now = time.time()
with mock.patch('time.time', new=lambda: now): with mock.patch('time.time', new=lambda: now):

121
test/unit/common/test_db.py
View File

@ -46,7 +46,7 @@ from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException from swift.common.swob import HTTPException
from test.unit import with_tempdir, make_timestamp_iter from test.unit import make_timestamp_iter, generate_db_path
class TestHelperFunctions(unittest.TestCase): class TestHelperFunctions(unittest.TestCase):
@ -180,10 +180,35 @@ class TestGreenDBConnection(unittest.TestCase):
InterceptConnection.commit.call_count)) InterceptConnection.commit.call_count))
class TestGetDBConnection(unittest.TestCase): class TestDbBase(unittest.TestCase):
server_type = 'container'
testdir = None
def setUp(self):
self.testdir = mkdtemp()
self.db_path = self.get_db_path()
def tearDown(self):
rmtree(self.testdir, ignore_errors=True)
def get_db_path(self):
return generate_db_path(self.testdir, self.server_type)
class TestGetDBConnection(TestDbBase):
def setUp(self):
super(TestGetDBConnection, self).setUp()
self.db_path = self.init_db_path()
def init_db_path(self):
# Test ContainerBroker.empty
db_path = self.get_db_path()
broker = ExampleBroker(db_path, account='a')
broker.initialize(Timestamp.now().internal, 0)
return db_path
def test_normal_case(self): def test_normal_case(self):
conn = get_db_connection(':memory:') conn = get_db_connection(self.db_path)
self.assertTrue(hasattr(conn, 'execute')) self.assertTrue(hasattr(conn, 'execute'))
def test_invalid_path(self): def test_invalid_path(self):
@ -201,8 +226,8 @@ class TestGetDBConnection(unittest.TestCase):
InterceptCursor.execute = mock_db_cmd InterceptCursor.execute = mock_db_cmd
with patch('sqlite3.Cursor', new=InterceptCursor): with patch('sqlite3.Cursor', new=InterceptCursor):
self.assertRaises(Timeout, get_db_connection, ':memory:', self.assertRaises(Timeout, get_db_connection,
timeout=0.1) self.db_path, timeout=0.1)
self.assertTrue(mock_db_cmd.called) self.assertTrue(mock_db_cmd.called)
self.assertEqual(mock_db_cmd.call_args_list, self.assertEqual(mock_db_cmd.call_args_list,
list((mock_db_cmd.call_args,) * list((mock_db_cmd.call_args,) *
@ -319,7 +344,7 @@ class ExampleBroker(DatabaseBroker):
Timestamp(info['put_timestamp'])) Timestamp(info['put_timestamp']))
class TestExampleBroker(unittest.TestCase): class TestExampleBroker(TestDbBase):
""" """
Tests that use the mostly Concrete enough ExampleBroker to exercise some Tests that use the mostly Concrete enough ExampleBroker to exercise some
of the abstract methods on DatabaseBroker. of the abstract methods on DatabaseBroker.
@ -327,19 +352,21 @@ class TestExampleBroker(unittest.TestCase):
broker_class = ExampleBroker broker_class = ExampleBroker
policy = 0 policy = 0
server_type = 'example'
def setUp(self): def setUp(self):
super(TestExampleBroker, self).setUp()
self.ts = make_timestamp_iter() self.ts = make_timestamp_iter()
def test_delete_db(self): def test_delete_db(self):
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal) broker.initialize(next(self.ts).internal)
broker.delete_db(next(self.ts).internal) broker.delete_db(next(self.ts).internal)
self.assertTrue(broker.is_deleted()) self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_delete(self): def test_merge_timestamps_simple_delete(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp) broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at'] created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0') broker.merge_timestamps(created_at, put_timestamp, '0')
@ -366,7 +393,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_delete_with_objects(self): def test_merge_timestamps_delete_with_objects(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy)) broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at'] created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0') broker.merge_timestamps(created_at, put_timestamp, '0')
@ -397,7 +424,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_simple_recreate(self): def test_merge_timestamps_simple_recreate(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy)) broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at'] virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at'] created_at = broker.get_info()['created_at']
@ -425,7 +452,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_recreate_with_objects(self): def test_merge_timestamps_recreate_with_objects(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy)) broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at'] created_at = broker.get_info()['created_at']
# delete # delete
@ -459,7 +486,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_put_no_status_change(self): def test_merge_timestamps_update_put_no_status_change(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy)) broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info() info = broker.get_info()
orig_status_changed_at = info['status_changed_at'] orig_status_changed_at = info['status_changed_at']
@ -472,7 +499,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_delete_no_status_change(self): def test_merge_timestamps_update_delete_no_status_change(self):
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy)) broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at'] created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, broker.merge_timestamps(created_at, put_timestamp,
@ -486,19 +513,23 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(orig_status_changed_at, info['status_changed_at']) self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_get_max_row(self): def test_get_max_row(self):
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal, broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
self.assertEqual(-1, broker.get_max_row()) self.assertEqual(-1, broker.get_max_row())
self.put_item(broker, next(self.ts).internal) self.put_item(broker, next(self.ts).internal)
# commit pending file into db
broker._commit_puts()
self.assertEqual(1, broker.get_max_row()) self.assertEqual(1, broker.get_max_row())
self.delete_item(broker, next(self.ts).internal) self.delete_item(broker, next(self.ts).internal)
broker._commit_puts()
self.assertEqual(2, broker.get_max_row()) self.assertEqual(2, broker.get_max_row())
self.put_item(broker, next(self.ts).internal) self.put_item(broker, next(self.ts).internal)
broker._commit_puts()
self.assertEqual(3, broker.get_max_row()) self.assertEqual(3, broker.get_max_row())
def test_get_info(self): def test_get_info(self):
broker = self.broker_class(':memory:', account='test', container='c') broker = self.broker_class(self.db_path, account='test', container='c')
created_at = time.time() created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at): with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(Timestamp(1).internal, broker.initialize(Timestamp(1).internal,
@ -518,7 +549,7 @@ class TestExampleBroker(unittest.TestCase):
k, info[k], v)) k, info[k], v))
def test_get_raw_metadata(self): def test_get_raw_metadata(self):
broker = self.broker_class(':memory:', account='test', container='c') broker = self.broker_class(self.db_path, account='test', container='c')
broker.initialize(Timestamp(0).internal, broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {}) self.assertEqual(broker.metadata, {})
@ -543,7 +574,7 @@ class TestExampleBroker(unittest.TestCase):
json.dumps(metadata)) json.dumps(metadata))
def test_put_timestamp(self): def test_put_timestamp(self):
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
orig_put_timestamp = next(self.ts).internal orig_put_timestamp = next(self.ts).internal
broker.initialize(orig_put_timestamp, broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
@ -564,7 +595,7 @@ class TestExampleBroker(unittest.TestCase):
newer_put_timestamp) newer_put_timestamp)
def test_status_changed_at(self): def test_status_changed_at(self):
broker = self.broker_class(':memory:', account='test', container='c') broker = self.broker_class(self.db_path, account='test', container='c')
put_timestamp = next(self.ts).internal put_timestamp = next(self.ts).internal
created_at = time.time() created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at): with patch('swift.common.db.time.time', new=lambda: created_at):
@ -590,7 +621,7 @@ class TestExampleBroker(unittest.TestCase):
status_changed_at) status_changed_at)
def test_get_syncs(self): def test_get_syncs(self):
broker = self.broker_class(':memory:', account='a', container='c') broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(Timestamp.now().internal, broker.initialize(Timestamp.now().internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
self.assertEqual([], broker.get_syncs()) self.assertEqual([], broker.get_syncs())
@ -603,9 +634,8 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}], self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}],
broker.get_syncs(incoming=False)) broker.get_syncs(incoming=False))
@with_tempdir def test_commit_pending(self):
def test_commit_pending(self, tempdir): broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
account='a', container='c') account='a', container='c')
broker.initialize(next(self.ts).internal, broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
@ -616,12 +646,12 @@ class TestExampleBroker(unittest.TestCase):
info = rows[0] info = rows[0]
count_key = '%s_count' % broker.db_contains_type count_key = '%s_count' % broker.db_contains_type
self.assertEqual(0, info[count_key]) self.assertEqual(0, info[count_key])
broker.get_info() # commit pending file into db
broker._commit_puts()
self.assertEqual(1, broker.get_info()[count_key]) self.assertEqual(1, broker.get_info()[count_key])
@with_tempdir def test_maybe_get(self):
def test_maybe_get(self, tempdir): broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
account='a', container='c') account='a', container='c')
broker.initialize(next(self.ts).internal, broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
@ -639,13 +669,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(broker.conn, conn) self.assertEqual(broker.conn, conn)
class TestDatabaseBroker(unittest.TestCase): class TestDatabaseBroker(TestDbBase):
def setUp(self):
self.testdir = mkdtemp()
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
def test_DB_PREALLOCATION_setting(self): def test_DB_PREALLOCATION_setting(self):
u = uuid4().hex u = uuid4().hex
@ -656,8 +680,8 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertRaises(OSError, b._preallocate) self.assertRaises(OSError, b._preallocate)
def test_memory_db_init(self): def test_memory_db_init(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
self.assertEqual(broker.db_file, ':memory:') self.assertEqual(broker.db_file, self.db_path)
self.assertRaises(AttributeError, broker.initialize, self.assertRaises(AttributeError, broker.initialize,
normalize_timestamp('0')) normalize_timestamp('0'))
@ -686,7 +710,7 @@ class TestDatabaseBroker(unittest.TestCase):
def test_initialize(self): def test_initialize(self):
self.assertRaises(AttributeError, self.assertRaises(AttributeError,
DatabaseBroker(':memory:').initialize, DatabaseBroker(self.db_path).initialize,
normalize_timestamp('1')) normalize_timestamp('1'))
stub_dict = {} stub_dict = {}
@ -694,7 +718,7 @@ class TestDatabaseBroker(unittest.TestCase):
stub_dict.clear() stub_dict.clear()
stub_dict['args'] = args stub_dict['args'] = args
stub_dict.update(kwargs) stub_dict.update(kwargs)
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
broker._initialize = stub broker._initialize = stub
broker.initialize(normalize_timestamp('1')) broker.initialize(normalize_timestamp('1'))
self.assertTrue(hasattr(stub_dict['args'][0], 'execute')) self.assertTrue(hasattr(stub_dict['args'][0], 'execute'))
@ -734,7 +758,7 @@ class TestDatabaseBroker(unittest.TestCase):
def do_test(expected_metadata, delete_meta_whitelist=None): def do_test(expected_metadata, delete_meta_whitelist=None):
if not delete_meta_whitelist: if not delete_meta_whitelist:
delete_meta_whitelist = [] delete_meta_whitelist = []
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.get_db_path())
broker.delete_meta_whitelist = delete_meta_whitelist broker.delete_meta_whitelist = delete_meta_whitelist
broker.db_type = 'test' broker.db_type = 'test'
broker._initialize = init_stub broker._initialize = init_stub
@ -790,13 +814,13 @@ class TestDatabaseBroker(unittest.TestCase):
['x-container-meta-test', 'x-something-else']) ['x-container-meta-test', 'x-something-else'])
def test_get(self): def test_get(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
with self.assertRaises(DatabaseConnectionError) as raised, \ with self.assertRaises(DatabaseConnectionError) as raised, \
broker.get() as conn: broker.get() as conn:
conn.execute('SELECT 1') conn.execute('SELECT 1')
self.assertEqual( self.assertEqual(
str(raised.exception), str(raised.exception),
"DB connection error (:memory:, 0):\nDB doesn't exist") "DB connection error (%s, 0):\nDB doesn't exist" % self.db_path)
broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker = DatabaseBroker(os.path.join(self.testdir, '1.db'))
with self.assertRaises(DatabaseConnectionError) as raised, \ with self.assertRaises(DatabaseConnectionError) as raised, \
@ -932,7 +956,7 @@ class TestDatabaseBroker(unittest.TestCase):
pass pass
def test_newid(self): def test_newid(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
broker.db_type = 'test' broker.db_type = 'test'
broker.db_contains_type = 'test' broker.db_contains_type = 'test'
uuid1 = str(uuid4()) uuid1 = str(uuid4())
@ -983,7 +1007,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(points[0][1], uuid2) self.assertEqual(points[0][1], uuid2)
def test_get_items_since(self): def test_get_items_since(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
broker.db_type = 'test' broker.db_type = 'test'
broker.db_contains_type = 'test' broker.db_contains_type = 'test'
@ -1005,7 +1029,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_items_since(999, 2), []) self.assertEqual(broker.get_items_since(999, 2), [])
def test_get_sync(self): def test_get_sync(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
broker.db_type = 'test' broker.db_type = 'test'
broker.db_contains_type = 'test' broker.db_contains_type = 'test'
uuid1 = str(uuid4()) uuid1 = str(uuid4())
@ -1045,7 +1069,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_sync(uuid3, incoming=False), 2) self.assertEqual(broker.get_sync(uuid3, incoming=False), 2)
def test_merge_syncs(self): def test_merge_syncs(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
def stub(*args, **kwargs): def stub(*args, **kwargs):
pass pass
@ -1090,7 +1114,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.get_replication_info_tester(metadata=True) self.get_replication_info_tester(metadata=True)
def get_replication_info_tester(self, metadata=False): def get_replication_info_tester(self, metadata=False):
broker = DatabaseBroker(':memory:', account='a') broker = DatabaseBroker(self.db_path, account='a')
broker.db_type = 'test' broker.db_type = 'test'
broker.db_contains_type = 'test' broker.db_contains_type = 'test'
broker.db_reclaim_timestamp = 'created_at' broker.db_reclaim_timestamp = 'created_at'
@ -1413,7 +1437,7 @@ class TestDatabaseBroker(unittest.TestCase):
(dbpath, qpath, hint)) (dbpath, qpath, hint))
def test_skip_commits(self): def test_skip_commits(self):
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(self.db_path)
self.assertTrue(broker._skip_commit_puts()) self.assertTrue(broker._skip_commit_puts())
broker._initialize = MagicMock() broker._initialize = MagicMock()
broker.initialize(Timestamp.now()) broker.initialize(Timestamp.now())
@ -1570,7 +1594,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertFalse(pending) self.assertFalse(pending)
class TestTombstoneReclaimer(unittest.TestCase): class TestTombstoneReclaimer(TestDbBase):
def _make_object(self, broker, obj_name, ts, deleted): def _make_object(self, broker, obj_name, ts, deleted):
if deleted: if deleted:
broker.delete_test(obj_name, ts.internal) broker.delete_test(obj_name, ts.internal)
@ -1588,7 +1612,8 @@ class TestTombstoneReclaimer(unittest.TestCase):
return self._count_reclaimable(conn, reclaim_age) return self._count_reclaimable(conn, reclaim_age)
def _setup_tombstones(self, reverse_names=True): def _setup_tombstones(self, reverse_names=True):
broker = ExampleBroker(':memory:', account='test_account', broker = ExampleBroker(self.db_path,
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
now = time.time() now = time.time()

233
test/unit/container/test_backend.py
View File

@ -53,13 +53,15 @@ from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
from test.unit.common import test_db from test.unit.common import test_db
class TestContainerBroker(unittest.TestCase): class TestContainerBroker(test_db.TestDbBase):
"""Tests for ContainerBroker""" """Tests for ContainerBroker"""
expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object', expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object',
'sqlite_sequence', 'policy_stat', 'sqlite_sequence', 'policy_stat',
'container_info', 'shard_range'} 'container_info', 'shard_range'}
server_type = 'container'
def setUp(self): def setUp(self):
super(TestContainerBroker, self).setUp()
self.ts = make_timestamp_iter() self.ts = make_timestamp_iter()
def _assert_shard_ranges(self, broker, expected, include_own=False): def _assert_shard_ranges(self, broker, expected, include_own=False):
@ -70,8 +72,9 @@ class TestContainerBroker(unittest.TestCase):
def test_creation(self): def test_creation(self):
# Test ContainerBroker.__init__ # Test ContainerBroker.__init__
broker = ContainerBroker(':memory:', account='a', container='c') db_file = self.get_db_path()
self.assertEqual(broker._db_file, ':memory:') broker = ContainerBroker(db_file, account='a', container='c')
self.assertEqual(broker._db_file, db_file)
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
curs = conn.cursor() curs = conn.cursor()
@ -83,6 +86,8 @@ class TestContainerBroker(unittest.TestCase):
# check the update trigger # check the update trigger
broker.put_object('blah', Timestamp.now().internal, 0, 'text/plain', broker.put_object('blah', Timestamp.now().internal, 0, 'text/plain',
'etag', 0, 0) 'etag', 0, 0)
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
with self.assertRaises(sqlite3.DatabaseError) as cm: with self.assertRaises(sqlite3.DatabaseError) as cm:
conn.execute('UPDATE object SET name="blah";') conn.execute('UPDATE object SET name="blah";')
@ -98,7 +103,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies @patch_policies
def test_storage_policy_property(self): def test_storage_policy_property(self):
for policy in POLICIES: for policy in POLICIES:
broker = ContainerBroker(':memory:', account='a', broker = ContainerBroker(self.get_db_path(), account='a',
container='policy_%s' % policy.name) container='policy_%s' % policy.name)
broker.initialize(next(self.ts).internal, policy.idx) broker.initialize(next(self.ts).internal, policy.idx)
with broker.get() as conn: with broker.get() as conn:
@ -121,7 +126,8 @@ class TestContainerBroker(unittest.TestCase):
# Test ContainerBroker throwing a conn away after # Test ContainerBroker throwing a conn away after
# unhandled exception # unhandled exception
first_conn = None first_conn = None
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
first_conn = conn first_conn = conn
@ -618,11 +624,14 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual('.shards_a/cc', broker.root_path) self.assertEqual('.shards_a/cc', broker.root_path)
def test_reclaim(self): def test_reclaim(self):
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain', broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM object " "SELECT count(*) FROM object "
@ -640,6 +649,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0) "WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001) sleep(.00001)
broker.delete_object('o', Timestamp.now().internal) broker.delete_object('o', Timestamp.now().internal)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM object " "SELECT count(*) FROM object "
@ -673,6 +683,7 @@ class TestContainerBroker(unittest.TestCase):
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('z', Timestamp.now().internal, 0, 'text/plain', broker.put_object('z', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
broker._commit_puts()
# Test before deletion # Test before deletion
broker.reclaim(Timestamp.now().internal, time()) broker.reclaim(Timestamp.now().internal, time())
broker.delete_db(Timestamp.now().internal) broker.delete_db(Timestamp.now().internal)
@ -689,7 +700,8 @@ class TestContainerBroker(unittest.TestCase):
random.seed(now) random.seed(now)
random.shuffle(obj_specs) random.shuffle(obj_specs)
policy_indexes = list(p.idx for p in POLICIES) policy_indexes = list(p.idx for p in POLICIES)
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
for i, obj_spec in enumerate(obj_specs): for i, obj_spec in enumerate(obj_specs):
@ -703,6 +715,8 @@ class TestContainerBroker(unittest.TestCase):
else: else:
broker.put_object(obj_name, ts.internal, 0, 'text/plain', broker.put_object(obj_name, ts.internal, 0, 'text/plain',
'etag', storage_policy_index=pidx) 'etag', storage_policy_index=pidx)
# commit pending file into db
broker._commit_puts()
def count_reclaimable(conn, reclaim_age): def count_reclaimable(conn, reclaim_age):
return conn.execute( return conn.execute(
@ -749,7 +763,8 @@ class TestContainerBroker(unittest.TestCase):
trace[1][2] > trace[2][2]) trace[1][2] > trace[2][2])
def test_reclaim_with_duplicate_names(self): def test_reclaim_with_duplicate_names(self):
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
now = time() now = time()
@ -758,6 +773,8 @@ class TestContainerBroker(unittest.TestCase):
for spidx in range(10): for spidx in range(10):
obj_name = 'object%s' % i obj_name = 'object%s' % i
broker.delete_object(obj_name, ages_ago.internal, spidx) broker.delete_object(obj_name, ages_ago.internal, spidx)
# commit pending file into db
broker._commit_puts()
reclaim_age = now - (2 * 7 * 24 * 60 * 60) reclaim_age = now - (2 * 7 * 24 * 60 * 60)
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
@ -844,7 +861,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info_is_deleted(self): def test_get_info_is_deleted(self):
ts = make_timestamp_iter() ts = make_timestamp_iter()
start = next(ts) start = next(ts)
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
# create it # create it
broker.initialize(start.internal, POLICIES.default.idx) broker.initialize(start.internal, POLICIES.default.idx)
@ -893,10 +911,13 @@ class TestContainerBroker(unittest.TestCase):
def test_delete_object(self): def test_delete_object(self):
# Test ContainerBroker.delete_object # Test ContainerBroker.delete_object
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain', broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM object " "SELECT count(*) FROM object "
@ -906,6 +927,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0) "WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001) sleep(.00001)
broker.delete_object('o', Timestamp.now().internal) broker.delete_object('o', Timestamp.now().internal)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT count(*) FROM object " "SELECT count(*) FROM object "
@ -916,7 +938,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object(self): def test_put_object(self):
# Test ContainerBroker.put_object # Test ContainerBroker.put_object
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
# Create initial object # Create initial object
@ -924,6 +947,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123, broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test', 'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe') '5af83e3196bf99f440f31f2e1a6c9afe')
# commit pending file into db
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -945,6 +970,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123, broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test', 'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe') '5af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -968,6 +994,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 124, broker.put_object('"{<object \'&\' name>}"', timestamp, 124,
'application/x-test', 'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449') 'aa0749bacbc79ec65fe206943d8fe449')
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -990,6 +1017,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', otimestamp, 124, broker.put_object('"{<object \'&\' name>}"', otimestamp, 124,
'application/x-test', 'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449') 'aa0749bacbc79ec65fe206943d8fe449')
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -1011,6 +1039,7 @@ class TestContainerBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '', broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '',
deleted=1) deleted=1)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -1033,6 +1062,7 @@ class TestContainerBroker(unittest.TestCase):
timestamp = Timestamp.now().internal timestamp = Timestamp.now().internal
broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '', broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '',
deleted=1) deleted=1)
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -1048,6 +1078,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123, broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test', 'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe') '5af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -1096,6 +1127,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 456, broker.put_object('"{<object \'&\' name>}"', timestamp, 456,
'application/x-test3', 'application/x-test3',
'6af83e3196bf99f440f31f2e1a6c9afe') '6af83e3196bf99f440f31f2e1a6c9afe')
broker._commit_puts()
with broker.get() as conn: with broker.get() as conn:
self.assertEqual(conn.execute( self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0], "SELECT name FROM object").fetchone()[0],
@ -1115,7 +1147,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_range_single_record(self): def test_merge_shard_range_single_record(self):
# Test ContainerBroker.merge_shard_range # Test ContainerBroker.merge_shard_range
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
# Stash these for later # Stash these for later
@ -1420,7 +1453,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_ranges_deleted(self): def test_merge_shard_ranges_deleted(self):
# Test ContainerBroker.merge_shard_ranges sets deleted attribute # Test ContainerBroker.merge_shard_ranges sets deleted attribute
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
# put shard range # put shard range
broker.merge_shard_ranges(ShardRange('a/o', next(self.ts).internal)) broker.merge_shard_ranges(ShardRange('a/o', next(self.ts).internal))
@ -1453,7 +1487,8 @@ class TestContainerBroker(unittest.TestCase):
'storage_policy_index': '2', 'storage_policy_index': '2',
'ctype_timestamp': None, 'ctype_timestamp': None,
'meta_timestamp': None} 'meta_timestamp': None}
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
expect = ('obj', '1234567890.12345', 42, 'text/plain', 'hash_test', expect = ('obj', '1234567890.12345', 42, 'text/plain', 'hash_test',
'1', '2', None, None) '1', '2', None, None)
@ -1627,7 +1662,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_encoded_timestamps_using_memory(self): def test_put_object_multiple_encoded_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type # Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps # and metadata timestamps
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
self._test_put_object_multiple_encoded_timestamps(broker) self._test_put_object_multiple_encoded_timestamps(broker)
@with_tempdir @with_tempdir
@ -1961,7 +1997,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_explicit_timestamps_using_memory(self): def test_put_object_multiple_explicit_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type # Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps passed as explicit args # and metadata timestamps passed as explicit args
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
self._test_put_object_multiple_explicit_timestamps(broker) self._test_put_object_multiple_explicit_timestamps(broker)
@with_tempdir @with_tempdir
@ -1977,7 +2014,8 @@ class TestContainerBroker(unittest.TestCase):
# Test container listing reports the most recent of data or metadata # Test container listing reports the most recent of data or metadata
# timestamp as last-modified time # timestamp as last-modified time
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(next(ts).internal, 0) broker.initialize(next(ts).internal, 0)
# simple 'single' timestamp case # simple 'single' timestamp case
@ -2030,7 +2068,7 @@ class TestContainerBroker(unittest.TestCase):
def test_put_misplaced_object_does_not_effect_container_stats(self): def test_put_misplaced_object_does_not_effect_container_stats(self):
policy = random.choice(list(POLICIES)) policy = random.choice(list(POLICIES))
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', broker = ContainerBroker(self.get_db_path(),
account='a', container='c') account='a', container='c')
broker.initialize(next(ts).internal, policy.idx) broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize # migration tests may not honor policy on initialize
@ -2057,7 +2095,7 @@ class TestContainerBroker(unittest.TestCase):
def test_has_multiple_policies(self): def test_has_multiple_policies(self):
policy = random.choice(list(POLICIES)) policy = random.choice(list(POLICIES))
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', broker = ContainerBroker(self.get_db_path(),
account='a', container='c') account='a', container='c')
broker.initialize(next(ts).internal, policy.idx) broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize # migration tests may not honor policy on initialize
@ -2069,18 +2107,21 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain', broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe', '5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx) storage_policy_index=policy.idx)
# commit pending file into db
broker._commit_puts()
self.assertFalse(broker.has_multiple_policies()) self.assertFalse(broker.has_multiple_policies())
other_policy = [p for p in POLICIES if p is not policy][0] other_policy = [p for p in POLICIES if p is not policy][0]
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain', broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe', '5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx) storage_policy_index=other_policy.idx)
broker._commit_puts()
self.assertTrue(broker.has_multiple_policies()) self.assertTrue(broker.has_multiple_policies())
@patch_policies @patch_policies
def test_get_policy_info(self): def test_get_policy_info(self):
policy = random.choice(list(POLICIES)) policy = random.choice(list(POLICIES))
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', broker = ContainerBroker(self.get_db_path(),
account='a', container='c') account='a', container='c')
broker.initialize(next(ts).internal, policy.idx) broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize # migration tests may not honor policy on initialize
@ -2097,6 +2138,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain', broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe', '5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx) storage_policy_index=policy.idx)
# commit pending file into db
broker._commit_puts()
policy_stats = broker.get_policy_stats() policy_stats = broker.get_policy_stats()
expected = {policy.idx: {'bytes_used': 123, 'object_count': 1}} expected = {policy.idx: {'bytes_used': 123, 'object_count': 1}}
self.assertEqual(policy_stats, expected) self.assertEqual(policy_stats, expected)
@ -2107,6 +2150,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain', broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe', '5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx) storage_policy_index=other_policy.idx)
broker._commit_puts()
policy_stats = broker.get_policy_stats() policy_stats = broker.get_policy_stats()
expected = { expected = {
policy.idx: {'bytes_used': 123, 'object_count': 1}, policy.idx: {'bytes_used': 123, 'object_count': 1},
@ -2117,7 +2161,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies @patch_policies
def test_policy_stat_tracking(self): def test_policy_stat_tracking(self):
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', broker = ContainerBroker(self.get_db_path(),
account='a', container='c') account='a', container='c')
# Note: in subclasses of this TestCase that inherit the # Note: in subclasses of this TestCase that inherit the
# ContainerBrokerMigrationMixin, passing POLICIES.default.idx here has # ContainerBrokerMigrationMixin, passing POLICIES.default.idx here has
@ -2149,6 +2193,8 @@ class TestContainerBroker(unittest.TestCase):
# track the size of the latest timestamp put for each object # track the size of the latest timestamp put for each object
# in each storage policy # in each storage policy
stats[policy_index][name] = size stats[policy_index][name] = size
# commit pending file into db
broker._commit_puts()
policy_stats = broker.get_policy_stats() policy_stats = broker.get_policy_stats()
if POLICIES.default.idx not in stats: if POLICIES.default.idx not in stats:
# unlikely, but check empty default index still in policy stats # unlikely, but check empty default index still in policy stats
@ -2161,7 +2207,7 @@ class TestContainerBroker(unittest.TestCase):
sum(stats[policy_index].values())) sum(stats[policy_index].values()))
def test_initialize_container_broker_in_default(self): def test_initialize_container_broker_in_default(self):
broker = ContainerBroker(':memory:', account='test1', broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2') container='test2')
# initialize with no storage_policy_index argument # initialize with no storage_policy_index argument
@ -2200,7 +2246,7 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info(self): def test_get_info(self):
# Test ContainerBroker.get_info # Test ContainerBroker.get_info
broker = ContainerBroker(':memory:', account='test1', broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2') container='test2')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
@ -2341,7 +2387,7 @@ class TestContainerBroker(unittest.TestCase):
'db_state': 'collapsed'}) 'db_state': 'collapsed'})
def test_set_x_syncs(self): def test_set_x_syncs(self):
broker = ContainerBroker(':memory:', account='test1', broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2') container='test2')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
@ -2355,7 +2401,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(info['x_container_sync_point2'], 2) self.assertEqual(info['x_container_sync_point2'], 2)
def test_get_report_info(self): def test_get_report_info(self):
broker = ContainerBroker(':memory:', account='test1', broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2') container='test2')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
@ -2527,7 +2573,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertFalse(get_rows(broker)) self.assertFalse(get_rows(broker))
def test_get_objects(self): def test_get_objects(self):
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
objects_0 = [{'name': 'obj_0_%d' % i, objects_0 = [{'name': 'obj_0_%d' % i,
'created_at': next(self.ts).normal, 'created_at': next(self.ts).normal,
@ -2575,7 +2622,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(objects_0 + objects_1, actual) self.assertEqual(objects_0 + objects_1, actual)
def test_get_objects_since_row(self): def test_get_objects_since_row(self):
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
obj_names = ['obj%03d' % i for i in range(20)] obj_names = ['obj%03d' % i for i in range(20)]
timestamps = [next(self.ts) for o in obj_names] timestamps = [next(self.ts) for o in obj_names]
@ -2625,7 +2673,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter(self): def test_list_objects_iter(self):
# Test ContainerBroker.list_objects_iter # Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4): for obj1 in range(4):
for obj2 in range(125): for obj2 in range(125):
@ -2775,7 +2824,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001']) self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_with_reserved_name(self): def test_list_objects_iter_with_reserved_name(self):
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(next(self.ts).internal, 0) broker.initialize(next(self.ts).internal, 0)
broker.put_object( broker.put_object(
@ -2934,10 +2984,13 @@ class TestContainerBroker(unittest.TestCase):
} }
failures = [] failures = []
for expected in expectations: for expected in expectations:
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, 0) broker.initialize(next(ts).internal, 0)
for name in expected['objects']: for name in expected['objects']:
broker.put_object(name, next(ts).internal, **obj_create_params) broker.put_object(name, next(ts).internal, **obj_create_params)
# commit pending file into db
broker._commit_puts()
params = default_listing_params.copy() params = default_listing_params.copy()
params.update(expected['params']) params.update(expected['params'])
listing = list(o[0] for o in broker.list_objects_iter(**params)) listing = list(o[0] for o in broker.list_objects_iter(**params))
@ -2952,7 +3005,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_non_slash(self): def test_list_objects_iter_non_slash(self):
# Test ContainerBroker.list_objects_iter using a # Test ContainerBroker.list_objects_iter using a
# delimiter that is not a slash # delimiter that is not a slash
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4): for obj1 in range(4):
for obj2 in range(125): for obj2 in range(125):
@ -3069,7 +3123,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_prefix_delim(self): def test_list_objects_iter_prefix_delim(self):
# Test ContainerBroker.list_objects_iter # Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object( broker.put_object(
@ -3108,7 +3163,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self): def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter # Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object( broker.put_object(
@ -3150,7 +3206,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self): def test_double_check_trailing_delimiter(self):
# Test ContainerBroker.list_objects_iter for a # Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter # container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0, broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3230,7 +3287,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter_non_slash(self): def test_double_check_trailing_delimiter_non_slash(self):
# Test ContainerBroker.list_objects_iter for a # Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter # container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0, broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3313,7 +3371,8 @@ class TestContainerBroker(unittest.TestCase):
s = s.encode('utf8') s = s.encode('utf8')
return md5(s, usedforsecurity=False).hexdigest() return md5(s, usedforsecurity=False).hexdigest()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0, broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3365,7 +3424,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_items_since(self): def test_get_items_since(self):
# test DatabaseBroker.get_items_since # test DatabaseBroker.get_items_since
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0, broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@ -3378,9 +3438,11 @@ class TestContainerBroker(unittest.TestCase):
def test_sync_merging(self): def test_sync_merging(self):
# exercise the DatabaseBroker sync functions a bit # exercise the DatabaseBroker sync functions a bit
broker1 = ContainerBroker(':memory:', account='a', container='c') broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
broker2 = ContainerBroker(':memory:', account='a', container='c') broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
self.assertEqual(broker2.get_sync('12345'), -1) self.assertEqual(broker2.get_sync('12345'), -1)
broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}])
@ -3388,14 +3450,18 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(broker2.get_sync('12345'), 3) self.assertEqual(broker2.get_sync('12345'), 3)
def test_merge_items(self): def test_merge_items(self):
broker1 = ContainerBroker(':memory:', account='a', container='c') broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
broker2 = ContainerBroker(':memory:', account='a', container='c') broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(1).internal, 0, broker1.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(2).internal, 0, broker1.put_object('b', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
id = broker1.get_info()['id'] id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
@ -3404,6 +3470,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_object('c', Timestamp(3).internal, 0, broker1.put_object('c', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000) items = broker2.get_items_since(-1, 1000)
@ -3431,19 +3498,24 @@ class TestContainerBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}' snowman = u'\N{SNOWMAN}'
if six.PY2: if six.PY2:
snowman = snowman.encode('utf-8') snowman = snowman.encode('utf-8')
broker1 = ContainerBroker(':memory:', account='a', container='c') broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id'] id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c') broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object(snowman, Timestamp(2).internal, 0, broker1.put_object(snowman, Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0, broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since( broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id) broker2.get_sync(id), 1000))), id)
broker1.put_object(snowman, Timestamp(4).internal, 0, 'text/plain', broker1.put_object(snowman, Timestamp(4).internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since( broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id) broker2.get_sync(id), 1000))), id)
items = broker2.get_items_since(-1, 1000) items = broker2.get_items_since(-1, 1000)
@ -3457,19 +3529,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_overwrite(self): def test_merge_items_overwrite(self):
# test DatabaseBroker.merge_items # test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c') broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id'] id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c') broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0, broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0, broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0, broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000) items = broker2.get_items_since(-1, 1000)
@ -3482,19 +3559,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_post_overwrite_out_of_order(self): def test_merge_items_post_overwrite_out_of_order(self):
# test DatabaseBroker.merge_items # test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c') broker1 = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker1.initialize(Timestamp('1').internal, 0) broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id'] id = broker1.get_info()['id']
broker2 = ContainerBroker(':memory:', account='a', container='c') broker2 = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0) broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0, broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0, broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
# commit pending file into db
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0, broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000) items = broker2.get_items_since(-1, 1000)
@ -3514,6 +3596,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(rec['created_at'], Timestamp(3).internal) self.assertEqual(rec['created_at'], Timestamp(3).internal)
broker1.put_object('b', Timestamp(5).internal, 0, broker1.put_object('b', Timestamp(5).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1._commit_puts()
broker2.merge_items(broker1.get_items_since( broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id) broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000) items = broker2.get_items_since(-1, 1000)
@ -3527,7 +3610,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index(self): def test_set_storage_policy_index(self):
ts = make_timestamp_iter() ts = make_timestamp_iter()
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
timestamp = next(ts) timestamp = next(ts)
broker.initialize(timestamp.internal, 0) broker.initialize(timestamp.internal, 0)
@ -3584,7 +3668,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index_empty(self): def test_set_storage_policy_index_empty(self):
# Putting an object may trigger migrations, so test with a # Putting an object may trigger migrations, so test with a
# never-had-an-object container to make sure we handle it # never-had-an-object container to make sure we handle it
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
info = broker.get_info() info = broker.get_info()
@ -3595,7 +3680,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(2, info['storage_policy_index']) self.assertEqual(2, info['storage_policy_index'])
def test_reconciler_sync(self): def test_reconciler_sync(self):
broker = ContainerBroker(':memory:', account='test_account', broker = ContainerBroker(self.get_db_path(),
account='test_account',
container='test_container') container='test_container')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
self.assertEqual(-1, broker.get_reconciler_sync()) self.assertEqual(-1, broker.get_reconciler_sync())
@ -5234,6 +5320,7 @@ class TestContainerBroker(unittest.TestCase):
class TestCommonContainerBroker(test_db.TestExampleBroker): class TestCommonContainerBroker(test_db.TestExampleBroker):
broker_class = ContainerBroker broker_class = ContainerBroker
server_type = 'container'
def setUp(self): def setUp(self):
super(TestCommonContainerBroker, self).setUp() super(TestCommonContainerBroker, self).setUp()
@ -5248,7 +5335,7 @@ class TestCommonContainerBroker(test_db.TestExampleBroker):
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
class ContainerBrokerMigrationMixin(object): class ContainerBrokerMigrationMixin(test_db.TestDbBase):
""" """
Mixin for running ContainerBroker against databases created with Mixin for running ContainerBroker against databases created with
older schemas. older schemas.
@ -5263,6 +5350,7 @@ class ContainerBrokerMigrationMixin(object):
return self.func.__get__(obj, obj_type) return self.func.__get__(obj, obj_type)
def setUp(self): def setUp(self):
super(ContainerBrokerMigrationMixin, self).setUp()
self._imported_create_object_table = \ self._imported_create_object_table = \
ContainerBroker.create_object_table ContainerBroker.create_object_table
ContainerBroker.create_object_table = \ ContainerBroker.create_object_table = \
@ -5303,6 +5391,7 @@ class ContainerBrokerMigrationMixin(object):
self._imported_create_shard_range_table self._imported_create_shard_range_table
ContainerBroker.create_policy_stat_table = \ ContainerBroker.create_policy_stat_table = \
self._imported_create_policy_stat_table self._imported_create_policy_stat_table
# We need to manually teardown and clean the self.tempdir
def premetadata_create_container_info_table(self, conn, put_timestamp, def premetadata_create_container_info_table(self, conn, put_timestamp,
@ -5359,7 +5448,8 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def setUp(self): def setUp(self):
super(TestContainerBrokerBeforeMetadata, self).setUp() super(TestContainerBrokerBeforeMetadata, self).setUp()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
exc = None exc = None
with broker.get() as conn: with broker.get() as conn:
@ -5371,10 +5461,12 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeMetadata, self).tearDown() super(TestContainerBrokerBeforeMetadata, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT metadata FROM container_stat') conn.execute('SELECT metadata FROM container_stat')
test_db.TestDbBase.tearDown(self)
def prexsync_create_container_info_table(self, conn, put_timestamp, def prexsync_create_container_info_table(self, conn, put_timestamp,
@ -5435,7 +5527,8 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
super(TestContainerBrokerBeforeXSync, self).setUp() super(TestContainerBrokerBeforeXSync, self).setUp()
ContainerBroker.create_container_info_table = \ ContainerBroker.create_container_info_table = \
prexsync_create_container_info_table prexsync_create_container_info_table
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
exc = None exc = None
with broker.get() as conn: with broker.get() as conn:
@ -5448,10 +5541,12 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeXSync, self).tearDown() super(TestContainerBrokerBeforeXSync, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT x_container_sync_point1 FROM container_stat') conn.execute('SELECT x_container_sync_point1 FROM container_stat')
test_db.TestDbBase.tearDown(self)
def prespi_create_object_table(self, conn, *args, **kwargs): def prespi_create_object_table(self, conn, *args, **kwargs):
@ -5552,7 +5647,8 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
ContainerBroker.create_container_info_table = \ ContainerBroker.create_container_info_table = \
prespi_create_container_info_table prespi_create_container_info_table
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \ with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn: broker.get() as conn:
@ -5563,10 +5659,12 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeSPI, self).tearDown() super(TestContainerBrokerBeforeSPI, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container_stat') conn.execute('SELECT storage_policy_index FROM container_stat')
test_db.TestDbBase.tearDown(self)
@patch_policies @patch_policies
@with_tempdir @with_tempdir
@ -5760,7 +5858,8 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def setUp(self): def setUp(self):
super(TestContainerBrokerBeforeShardRanges, self).setUp() super(TestContainerBrokerBeforeShardRanges, self).setUp()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \ with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn: broker.get() as conn:
@ -5770,11 +5869,13 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeShardRanges, self).tearDown() super(TestContainerBrokerBeforeShardRanges, self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('''SELECT * conn.execute('''SELECT *
FROM shard_range''') FROM shard_range''')
test_db.TestDbBase.tearDown(self)
def pre_reported_create_shard_range_table(self, conn): def pre_reported_create_shard_range_table(self, conn):
@ -5828,7 +5929,8 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
ContainerBroker.create_shard_range_table = \ ContainerBroker.create_shard_range_table = \
pre_reported_create_shard_range_table pre_reported_create_shard_range_table
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \ with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn: broker.get() as conn:
@ -5839,11 +5941,13 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeShardRangeReportedColumn, super(TestContainerBrokerBeforeShardRangeReportedColumn,
self).tearDown() self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('''SELECT reported conn.execute('''SELECT reported
FROM shard_range''') FROM shard_range''')
test_db.TestDbBase.tearDown(self)
@with_tempdir @with_tempdir
def test_get_shard_ranges_attempts(self, tempdir): def test_get_shard_ranges_attempts(self, tempdir):
@ -6056,7 +6160,8 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
ContainerBroker.create_shard_range_table = \ ContainerBroker.create_shard_range_table = \
pre_tombstones_create_shard_range_table pre_tombstones_create_shard_range_table
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \ with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn: broker.get() as conn:
@ -6067,11 +6172,13 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
def tearDown(self): def tearDown(self):
super(TestContainerBrokerBeforeShardRangeTombstonesColumn, super(TestContainerBrokerBeforeShardRangeTombstonesColumn,
self).tearDown() self).tearDown()
broker = ContainerBroker(':memory:', account='a', container='c') broker = ContainerBroker(self.get_db_path(), account='a',
container='c')
broker.initialize(Timestamp('1').internal, 0) broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn: with broker.get() as conn:
conn.execute('''SELECT tombstones conn.execute('''SELECT tombstones
FROM shard_range''') FROM shard_range''')
test_db.TestDbBase.tearDown(self)
class TestUpdateNewItemFromExisting(unittest.TestCase): class TestUpdateNewItemFromExisting(unittest.TestCase):