openstack/deb-swift
Code Issues Proposed changes

Merge "swift-[account|container]-info when disk is full"

Browse Source
This commit is contained in:
Jenkins 2016-04-22 17:31:08 +00:00 committed by Gerrit Code Review
commit 60c127575b
6 changed files with 164 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bin/swift-account-info
View File

@ -11,12 +11,28 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlite3
import sys
from optparse import OptionParser
from swift.cli.info import print_info, InfoSystemExit
def run_print_info(args, opts):
try:
print_info('account', *args, **opts)
except InfoSystemExit:
sys.exit(1)
except sqlite3.OperationalError as e:
if not opts.get('stale_reads_ok'):
opts['stale_reads_ok'] = True
print('Warning: Possibly Stale Data')
run_print_info(args, opts)
sys.exit(2)
else:
print('Account info failed: %s' % e)
sys.exit(1)
if __name__ == '__main__':
parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
parser.add_option(
@ -28,7 +44,4 @@ if __name__ == '__main__':
if len(args) != 1:
sys.exit(parser.print_help())
try:
print_info('account', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)
run_print_info(args, vars(options))

21
bin/swift-container-info
View File

@ -11,12 +11,28 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlite3
import sys
from optparse import OptionParser
from swift.cli.info import print_info, InfoSystemExit
def run_print_info(args, opts):
try:
print_info('container', *args, **opts)
except InfoSystemExit:
sys.exit(1)
except sqlite3.OperationalError as e:
if not opts.get('stale_reads_ok'):
opts['stale_reads_ok'] = True
print('Warning: Possibly Stale Data')
run_print_info(args, opts)
sys.exit(2)
else:
print('Container info failed: %s' % e)
sys.exit(1)
if __name__ == '__main__':
parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
parser.add_option(
@ -28,7 +44,4 @@ if __name__ == '__main__':
if len(args) != 1:
sys.exit(parser.print_help())
try:
print_info('container', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)
run_print_info(args, vars(options))

6
swift/cli/info.py
View File

@ -308,7 +308,7 @@ def print_obj_metadata(metadata):
print_metadata('Other Metadata:', other_metadata)
def print_info(db_type, db_file, swift_dir='/etc/swift'):
def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False):
if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error")
raise InfoSystemExit()
@ -318,10 +318,10 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file)
broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file)
broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = CBDATADIR
try:
info = broker.get_info()

2
swift/common/db.py
View File

@ -632,7 +632,7 @@ class DatabaseBroker(object):
with lock_parent_directory(self.pending_file,
self.pending_timeout):
self._commit_puts()
except LockTimeout:
except (LockTimeout, sqlite3.OperationalError):
if not self.stale_reads_ok:
raise

92
test/unit/account/test_backend.py
View File

@ -793,33 +793,79 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(items_by_name['b']['object_count'], 0)
self.assertEqual(items_by_name['b']['bytes_used'], 0)
def test_load_old_pending_puts(self):
@with_tempdir
def test_load_old_pending_puts(self, tempdir):
# pending puts from pre-storage-policy account brokers won't contain
# the storage policy index
tempdir = mkdtemp()
broker_path = os.path.join(tempdir, 'test-load-old.db')
try:
broker = AccountBroker(broker_path, account='real')
broker.initialize(Timestamp(1).internal)
with open(broker_path + '.pending', 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker = AccountBroker(broker_path, account='real')
broker.initialize(Timestamp(1).internal)
with open(broker.pending_file, 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts()
with broker.get() as conn:
results = list(conn.execute('''
SELECT name, storage_policy_index FROM container
'''))
self.assertEqual(len(results), 1)
self.assertEqual(dict(results[0]),
{'name': 'oldcon', 'storage_policy_index': 0})
finally:
rmtree(tempdir)
broker._commit_puts()
with broker.get() as conn:
results = list(conn.execute('''
SELECT name, storage_policy_index FROM container
'''))
self.assertEqual(len(results), 1)
self.assertEqual(dict(results[0]),
{'name': 'oldcon', 'storage_policy_index': 0})
@with_tempdir
def test_get_info_stale_read_ok(self, tempdir):
# test getting a stale read from the db
broker_path = os.path.join(tempdir, 'test-load-old.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = AccountBroker(broker_path, account='real',
stale_reads_ok=True)
broker.initialize(Timestamp(1).internal)
with open(broker.pending_file, 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts = mock_commit_puts
broker.get_info()
@with_tempdir
def test_get_info_no_stale_reads(self, tempdir):
broker_path = os.path.join(tempdir, 'test-load-old.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = AccountBroker(broker_path, account='real',
stale_reads_ok=False)
broker.initialize(Timestamp(1).internal)
with open(broker.pending_file, 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts = mock_commit_puts
with self.assertRaises(sqlite3.OperationalError) as exc_context:
broker.get_info()
self.assertIn('unable to open database file',
str(exc_context.exception))
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True),

57
test/unit/container/test_backend.py
View File

@ -2034,6 +2034,63 @@ class TestContainerBroker(unittest.TestCase):
}
self.assertEqual(broker.get_policy_stats(), expected)
@with_tempdir
def test_get_info_no_stale_reads(self, tempdir):
ts = (Timestamp(t).internal for t in
itertools.count(int(time())))
db_path = os.path.join(tempdir, 'container.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = ContainerBroker(db_path, account='a', container='c',
stale_reads_ok=False)
broker.initialize(next(ts), 1)
# manually make some pending entries
with open(broker.pending_file, 'a+b') as fp:
for i in range(10):
name, timestamp, size, content_type, etag, deleted = (
'o%s' % i, next(ts), 0, 'c', 'e', 0)
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, size, content_type, etag, deleted),
protocol=2).encode('base64'))
fp.flush()
broker._commit_puts = mock_commit_puts
with self.assertRaises(sqlite3.OperationalError) as exc_context:
broker.get_info()
self.assertIn('unable to open database file',
str(exc_context.exception))
@with_tempdir
def test_get_info_stale_read_ok(self, tempdir):
ts = (Timestamp(t).internal for t in
itertools.count(int(time())))
db_path = os.path.join(tempdir, 'container.db')
def mock_commit_puts():
raise sqlite3.OperationalError('unable to open database file')
broker = ContainerBroker(db_path, account='a', container='c',
stale_reads_ok=True)
broker.initialize(next(ts), 1)
# manually make some pending entries
with open(broker.pending_file, 'a+b') as fp:
for i in range(10):
name, timestamp, size, content_type, etag, deleted = (
'o%s' % i, next(ts), 0, 'c', 'e', 0)
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, size, content_type, etag, deleted),
protocol=2).encode('base64'))
fp.flush()
broker._commit_puts = mock_commit_puts
broker.get_info()
class TestCommonContainerBroker(test_db.TestExampleBroker):