swift-[account|container]-info when disk is full

Extended the use of the DatabaseBroker "stale_reads_ok" flag to the
AccountBroker and ContainerBroker.  Now checks for an sqlite3 error
from the _commit_puts call that processes the pending files.

If this error is raised, then the stale_reads_ok flag will be checked
to determine how to proceed as opposed to simply raising.

The first time that print_info is attempted, the flag will be
false, but swift-[account|container]-info will check for the
raised exception.  If it was raised, then a warning is reported
that the data may be stale, and another attempt will be
made using the stale_reads_ok=True flag.

Change-Id: I761526eef62327888c865d87a9caafa3e7eabab6
Closes-Bug: 1531302
This commit is contained in:
Janie Richling 2016-01-16 21:59:20 -06:00
parent 4db7e2e2e4
commit e97c4f794d
6 changed files with 164 additions and 35 deletions

View File

@ -11,12 +11,28 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import sqlite3
import sys import sys
from optparse import OptionParser from optparse import OptionParser
from swift.cli.info import print_info, InfoSystemExit from swift.cli.info import print_info, InfoSystemExit
def run_print_info(args, opts):
try:
print_info('account', *args, **opts)
except InfoSystemExit:
sys.exit(1)
except sqlite3.OperationalError as e:
if not opts.get('stale_reads_ok'):
opts['stale_reads_ok'] = True
print('Warning: Possibly Stale Data')
run_print_info(args, opts)
sys.exit(2)
else:
print('Account info failed: %s' % e)
sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser('%prog [options] ACCOUNT_DB_FILE') parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
parser.add_option( parser.add_option(
@ -28,7 +44,4 @@ if __name__ == '__main__':
if len(args) != 1: if len(args) != 1:
sys.exit(parser.print_help()) sys.exit(parser.print_help())
try: run_print_info(args, vars(options))
print_info('account', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)

View File

@ -11,12 +11,28 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import sqlite3
import sys import sys
from optparse import OptionParser from optparse import OptionParser
from swift.cli.info import print_info, InfoSystemExit from swift.cli.info import print_info, InfoSystemExit
def run_print_info(args, opts):
try:
print_info('container', *args, **opts)
except InfoSystemExit:
sys.exit(1)
except sqlite3.OperationalError as e:
if not opts.get('stale_reads_ok'):
opts['stale_reads_ok'] = True
print('Warning: Possibly Stale Data')
run_print_info(args, opts)
sys.exit(2)
else:
print('Container info failed: %s' % e)
sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser('%prog [options] CONTAINER_DB_FILE') parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
parser.add_option( parser.add_option(
@ -28,7 +44,4 @@ if __name__ == '__main__':
if len(args) != 1: if len(args) != 1:
sys.exit(parser.print_help()) sys.exit(parser.print_help())
try: run_print_info(args, vars(options))
print_info('container', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)

View File

@ -308,7 +308,7 @@ def print_obj_metadata(metadata):
print_metadata('Other Metadata:', other_metadata) print_metadata('Other Metadata:', other_metadata)
def print_info(db_type, db_file, swift_dir='/etc/swift'): def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False):
if db_type not in ('account', 'container'): if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error") print("Unrecognized DB type: internal error")
raise InfoSystemExit() raise InfoSystemExit()
@ -318,10 +318,10 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
if not db_file.startswith(('/', './')): if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account': if db_type == 'account':
broker = AccountBroker(db_file) broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = ABDATADIR datadir = ABDATADIR
else: else:
broker = ContainerBroker(db_file) broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = CBDATADIR datadir = CBDATADIR
try: try:
info = broker.get_info() info = broker.get_info()

View File

@ -628,7 +628,7 @@ class DatabaseBroker(object):
with lock_parent_directory(self.pending_file, with lock_parent_directory(self.pending_file,
self.pending_timeout): self.pending_timeout):
self._commit_puts() self._commit_puts()
except LockTimeout: except (LockTimeout, sqlite3.OperationalError):
if not self.stale_reads_ok: if not self.stale_reads_ok:
raise raise

View File

@ -793,15 +793,14 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(items_by_name['b']['object_count'], 0) self.assertEqual(items_by_name['b']['object_count'], 0)
self.assertEqual(items_by_name['b']['bytes_used'], 0) self.assertEqual(items_by_name['b']['bytes_used'], 0)
def test_load_old_pending_puts(self): @with_tempdir
def test_load_old_pending_puts(self, tempdir):
# pending puts from pre-storage-policy account brokers won't contain # pending puts from pre-storage-policy account brokers won't contain
# the storage policy index # the storage policy index
tempdir = mkdtemp()
broker_path = os.path.join(tempdir, 'test-load-old.db') broker_path = os.path.join(tempdir, 'test-load-old.db')
try:
broker = AccountBroker(broker_path, account='real') broker = AccountBroker(broker_path, account='real')
broker.initialize(Timestamp(1).internal) broker.initialize(Timestamp(1).internal)
with open(broker_path + '.pending', 'a+b') as pending: with open(broker.pending_file, 'a+b') as pending:
pending.write(':') pending.write(':')
pending.write(pickle.dumps( pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count, # name, put_timestamp, delete_timestamp, object_count,
@ -818,8 +817,55 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(dict(results[0]), self.assertEqual(dict(results[0]),
{'name': 'oldcon', 'storage_policy_index': 0}) {'name': 'oldcon', 'storage_policy_index': 0})
finally:
rmtree(tempdir) @with_tempdir
def test_get_info_stale_read_ok(self, tempdir):
# test getting a stale read from the db
broker_path = os.path.join(tempdir, 'test-load-old.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = AccountBroker(broker_path, account='real',
stale_reads_ok=True)
broker.initialize(Timestamp(1).internal)
with open(broker.pending_file, 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts = mock_commit_puts
broker.get_info()
@with_tempdir
def test_get_info_no_stale_reads(self, tempdir):
broker_path = os.path.join(tempdir, 'test-load-old.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = AccountBroker(broker_path, account='real',
stale_reads_ok=False)
broker.initialize(Timestamp(1).internal)
with open(broker.pending_file, 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts = mock_commit_puts
with self.assertRaises(sqlite3.OperationalError) as exc_context:
broker.get_info()
self.assertIn('unable to open database file',
str(exc_context.exception))
@patch_policies([StoragePolicy(0, 'zero', False), @patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True), StoragePolicy(1, 'one', True),

View File

@ -1679,6 +1679,63 @@ class TestContainerBroker(unittest.TestCase):
} }
self.assertEqual(broker.get_policy_stats(), expected) self.assertEqual(broker.get_policy_stats(), expected)
@with_tempdir
def test_get_info_no_stale_reads(self, tempdir):
ts = (Timestamp(t).internal for t in
itertools.count(int(time())))
db_path = os.path.join(tempdir, 'container.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = ContainerBroker(db_path, account='a', container='c',
stale_reads_ok=False)
broker.initialize(next(ts), 1)
# manually make some pending entries
with open(broker.pending_file, 'a+b') as fp:
for i in range(10):
name, timestamp, size, content_type, etag, deleted = (
'o%s' % i, next(ts), 0, 'c', 'e', 0)
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, size, content_type, etag, deleted),
protocol=2).encode('base64'))
fp.flush()
broker._commit_puts = mock_commit_puts
with self.assertRaises(sqlite3.OperationalError) as exc_context:
broker.get_info()
self.assertIn('unable to open database file',
str(exc_context.exception))
@with_tempdir
def test_get_info_stale_read_ok(self, tempdir):
ts = (Timestamp(t).internal for t in
itertools.count(int(time())))
db_path = os.path.join(tempdir, 'container.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = ContainerBroker(db_path, account='a', container='c',
stale_reads_ok=True)
broker.initialize(next(ts), 1)
# manually make some pending entries
with open(broker.pending_file, 'a+b') as fp:
for i in range(10):
name, timestamp, size, content_type, etag, deleted = (
'o%s' % i, next(ts), 0, 'c', 'e', 0)
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, size, content_type, etag, deleted),
protocol=2).encode('base64'))
fp.flush()
broker._commit_puts = mock_commit_puts
broker.get_info()
class TestCommonContainerBroker(test_db.TestExampleBroker): class TestCommonContainerBroker(test_db.TestExampleBroker):