recon: refactor common recon names into a common location

Change-Id: I0a0766cfb6672377de0f152ce179c874c327ec54
This commit is contained in:
Matthew Oliver 2021-03-31 15:07:39 +11:00 committed by Tim Burke
parent 8e1209d2b1
commit 85e36f7122
17 changed files with 139 additions and 50 deletions

View File

@ -24,6 +24,7 @@ from eventlet import Timeout
from swift.common.utils import get_logger, dump_recon_cache, readconf, \
lock_path
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.obj.diskfile import ASYNCDIR_BASE
@ -57,9 +58,9 @@ def main():
sys.exit(1)
conf = readconf(conf_path, 'filter:recon')
device_dir = conf.get('devices', '/srv/node')
recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift')
recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH)
recon_lock_path = conf.get('recon_lock_path', '/var/lock')
cache_file = os.path.join(recon_cache_path, "object.recon")
cache_file = os.path.join(recon_cache_path, RECON_OBJECT_FILE)
lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron")
conf['log_name'] = conf.get('log_name', 'recon-cron')
logger = get_logger(conf, log_route='recon-cron')

View File

@ -32,6 +32,7 @@ from swift.common.utils import replace_partition_in_path, config_true_value, \
non_negative_float, non_negative_int, config_auto_int_value, \
dump_recon_cache, get_partition_from_path
from swift.obj import diskfile
from swift.common.recon import RECON_RELINKER_FILE, DEFAULT_RECON_CACHE_PATH
LOCK_FILE = '.relink.{datadir}.lock'
@ -42,8 +43,6 @@ STEP_CLEANUP = 'cleanup'
EXIT_SUCCESS = 0
EXIT_NO_APPLICABLE_POLICY = 2
EXIT_ERROR = 1
RECON_FILE = 'relinker.recon'
DEFAULT_RECON_CACHE_PATH = '/var/cache/swift'
DEFAULT_STATS_INTERVAL = 300.0
@ -101,7 +100,7 @@ class Relinker(object):
def __init__(self, conf, logger, device_list=None, do_cleanup=False):
self.conf = conf
self.recon_cache = os.path.join(self.conf['recon_cache_path'],
RECON_FILE)
RECON_RELINKER_FILE)
self.logger = logger
self.device_list = device_list or []
self.do_cleanup = do_cleanup
@ -704,7 +703,7 @@ def parallel_process(do_cleanup, conf, logger=None, device_list=None):
# initialise recon dump for collection
# Lets start by always deleting last run's stats
recon_cache = os.path.join(conf['recon_cache_path'], RECON_FILE)
recon_cache = os.path.join(conf['recon_cache_path'], RECON_RELINKER_FILE)
_reset_recon(recon_cache, logger)
device_list = sorted(set(device_list or os.listdir(conf['devices'])))

View File

@ -25,11 +25,19 @@ from swift.common.utils import get_logger, audit_location_generator, \
config_true_value, dump_recon_cache, ratelimit_sleep
from swift.common.daemon import Daemon
from swift.common.exceptions import DatabaseAuditorException
from swift.common.recon import DEFAULT_RECON_CACHE_PATH, \
server_type_to_recon_file
class DatabaseAuditor(Daemon):
"""Base Database Auditor."""
@property
def rcache(self):
return os.path.join(
self.recon_cache_path,
server_type_to_recon_file(self.server_type))
@property
def server_type(self):
raise NotImplementedError
@ -54,10 +62,7 @@ class DatabaseAuditor(Daemon):
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path,
"{}.recon".format(self.server_type))
DEFAULT_RECON_CACHE_PATH)
self.datadir = '{}s'.format(self.server_type)
def _one_audit_pass(self, reported):

View File

@ -44,6 +44,8 @@ from swift.common.exceptions import DriveNotMounted
from swift.common.daemon import Daemon
from swift.common.swob import Response, HTTPNotFound, HTTPNoContent, \
HTTPAccepted, HTTPBadRequest
from swift.common.recon import DEFAULT_RECON_CACHE_PATH, \
server_type_to_recon_file
DEBUG_TIMINGS_THRESHOLD = 10
@ -229,8 +231,8 @@ class Replicator(Daemon):
config_true_value(conf.get('db_query_logging', 'f'))
self._zero_stats()
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.recon_replicator = '%s.recon' % self.server_type
DEFAULT_RECON_CACHE_PATH)
self.recon_replicator = server_type_to_recon_file(self.server_type)
self.rcache = os.path.join(self.recon_cache_path,
self.recon_replicator)
self.extract_device_re = re.compile('%s%s([^%s]+)' % (

View File

@ -25,6 +25,9 @@ from swift.common.constraints import check_mount
from swift.common.storage_policy import POLICIES
from swift.common.swob import Request, Response
from swift.common.utils import get_logger, SWIFT_CONF_FILE, md5_hash_for_file
from swift.common.recon import RECON_OBJECT_FILE, RECON_CONTAINER_FILE, \
RECON_ACCOUNT_FILE, RECON_DRIVE_FILE, RECON_RELINKER_FILE, \
DEFAULT_RECON_CACHE_PATH
class ReconMiddleware(object):
@ -47,17 +50,17 @@ class ReconMiddleware(object):
swift_dir = conf.get('swift_dir', '/etc/swift')
self.logger = get_logger(conf, log_route='recon')
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
DEFAULT_RECON_CACHE_PATH)
self.object_recon_cache = os.path.join(self.recon_cache_path,
'object.recon')
RECON_OBJECT_FILE)
self.container_recon_cache = os.path.join(self.recon_cache_path,
'container.recon')
RECON_CONTAINER_FILE)
self.account_recon_cache = os.path.join(self.recon_cache_path,
'account.recon')
RECON_ACCOUNT_FILE)
self.drive_recon_cache = os.path.join(self.recon_cache_path,
'drive.recon')
RECON_DRIVE_FILE)
self.relink_recon_cache = os.path.join(self.recon_cache_path,
"relinker.recon")
RECON_RELINKER_FILE)
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')

30
swift/common/recon.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright (c) 2010-2021 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import six
RECON_RELINKER_FILE = 'relinker.recon'
RECON_OBJECT_FILE = 'object.recon'
RECON_CONTAINER_FILE = 'container.recon'
RECON_ACCOUNT_FILE = 'account.recon'
RECON_DRIVE_FILE = 'drive.recon'
DEFAULT_RECON_CACHE_PATH = '/var/cache/swift'
def server_type_to_recon_file(server_type):
if not isinstance(server_type, six.string_types) or \
server_type.lower() not in ('account', 'container', 'object'):
raise ValueError('Invalid server_type')
return "%s.recon" % server_type.lower()

View File

@ -36,6 +36,7 @@ from swift.common.utils import get_logger, config_true_value, \
eventlet_monkey_patch
from swift.common.daemon import Daemon
from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
from swift.common.recon import RECON_CONTAINER_FILE, DEFAULT_RECON_CACHE_PATH
class ContainerUpdater(Daemon):
@ -75,8 +76,8 @@ class ContainerUpdater(Daemon):
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "container.recon")
DEFAULT_RECON_CACHE_PATH)
self.rcache = os.path.join(self.recon_cache_path, RECON_CONTAINER_FILE)
self.user_agent = 'container-updater %s' % os.getpid()
def get_account_ring(self):

View File

@ -33,6 +33,7 @@ from swift.common.utils import (
config_auto_int_value, dump_recon_cache, get_logger, list_from_csv,
listdir, load_pkg_resource, parse_prefixed_conf, ratelimit_sleep,
readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter)
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
class AuditorWorker(object):
@ -321,8 +322,8 @@ class ObjectAuditor(Daemon):
self.conf_zero_byte_fps = int(
conf.get('zero_byte_files_per_second', 50))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, "object.recon")
DEFAULT_RECON_CACHE_PATH)
self.rcache = join(self.recon_cache_path, RECON_OBJECT_FILE)
self.interval = float(conf.get('interval', 30))
watcher_names = set(list_from_csv(conf.get('watchers', '')))

View File

@ -33,6 +33,7 @@ from swift.common.utils import get_logger, dump_recon_cache, split_path, \
from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
HTTP_PRECONDITION_FAILED
from swift.common.swob import wsgi_quote, str_to_wsgi
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.container.reconciler import direct_delete_container_entry
@ -103,8 +104,8 @@ class ObjectExpirer(Daemon):
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
DEFAULT_RECON_CACHE_PATH)
self.rcache = join(self.recon_cache_path, RECON_OBJECT_FILE)
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")

View File

@ -38,6 +38,7 @@ from swift.common.utils import (
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.common.ring.utils import is_local_device
from swift.obj.ssync_sender import Sender as ssync_sender
from swift.common.http import HTTP_OK, HTTP_NOT_FOUND, \
@ -202,8 +203,8 @@ class ObjectReconstructor(Daemon):
self.http_timeout = int(conf.get('http_timeout', 60))
self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
DEFAULT_RECON_CACHE_PATH)
self.rcache = os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)
self._next_rcache_update = time.time() + self.stats_interval
# defaults subject to change after beta
self.conn_timeout = float(conf.get('conn_timeout', 0.5))

View File

@ -40,6 +40,7 @@ from swift.common.utils import whataremyips, unlink_older_than, \
from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.obj import ssync_sender
from swift.obj.diskfile import get_data_dir, get_tmp_dir, DiskFileRouter
from swift.common.storage_policy import POLICIES, REPL_POLICY
@ -172,8 +173,8 @@ class ObjectReplicator(Daemon):
self.rsync_module = '{replication_ip}::object'
self.http_timeout = int(conf.get('http_timeout', 60))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
DEFAULT_RECON_CACHE_PATH)
self.rcache = os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)
self._next_rcache_update = time.time() + self.stats_interval
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.node_timeout = float(conf.get('node_timeout', 10))

View File

@ -34,6 +34,7 @@ from swift.common.utils import get_logger, renamer, write_pickle, \
from swift.common.daemon import Daemon
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.storage_policy import split_policy_string, PolicyError
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.obj.diskfile import get_tmp_dir, ASYNCDIR_BASE
from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR, \
HTTP_MOVED_PERMANENTLY
@ -114,8 +115,8 @@ class ObjectUpdater(Daemon):
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.report_interval = float(conf.get('report_interval', 300))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, 'object.recon')
DEFAULT_RECON_CACHE_PATH)
self.rcache = os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)
self.stats = SweepStats()
def _listdir(self, path):

View File

@ -28,6 +28,8 @@ from unittest import TestCase
from swift import __version__ as swiftver
from swift.common import ring, utils
from swift.common.recon import RECON_RELINKER_FILE, RECON_DRIVE_FILE, \
DEFAULT_RECON_CACHE_PATH, server_type_to_recon_file
from swift.common.swob import Request
from swift.common.middleware import recon
from swift.common.storage_policy import StoragePolicy
@ -316,6 +318,11 @@ class TestReconSuccess(TestCase):
self._create_ring(ringpath, replica_map, self.ring_devs,
self.ring_part_shift)
def _full_recon_path(self, server_type, recon_file=None):
if server_type:
recon_file = server_type_to_recon_file(server_type)
return os.path.join(DEFAULT_RECON_CACHE_PATH, recon_file)
@patch_policies([
StoragePolicy(0, 'stagecoach'),
StoragePolicy(1, 'pinto', is_deprecated=True),
@ -683,7 +690,7 @@ class TestReconSuccess(TestCase):
rv = self.app.get_async_info()
self.assertEqual(self.fakecache.fakeout_calls,
[((['async_pending', 'async_pending_last'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, {'async_pending': 5, 'async_pending_last': now})
def test_get_replication_info_account(self):
@ -706,7 +713,7 @@ class TestReconSuccess(TestCase):
self.assertEqual(self.fakecache.fakeout_calls,
[((['replication_time', 'replication_stats',
'replication_last'],
'/var/cache/swift/account.recon'), {})])
self._full_recon_path('account')), {})])
self.assertEqual(rv, {
"replication_stats": {
"attempted": 1, "diff": 0,
@ -743,7 +750,8 @@ class TestReconSuccess(TestCase):
self.assertEqual(self.fakecache.fakeout_calls,
[((['replication_time', 'replication_stats',
'replication_last'],
'/var/cache/swift/container.recon'), {})])
self._full_recon_path('container')),
{})])
self.assertEqual(rv, {
"replication_time": 200.0,
"replication_stats": {
@ -780,7 +788,7 @@ class TestReconSuccess(TestCase):
[((['replication_time', 'replication_stats',
'replication_last', 'object_replication_time',
'object_replication_last'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, {
"replication_time": 0.2615511417388916,
"replication_stats": {
@ -806,7 +814,7 @@ class TestReconSuccess(TestCase):
rv = self.app.get_updater_info('container')
self.assertEqual(self.fakecache.fakeout_calls,
[((['container_updater_sweep'],
'/var/cache/swift/container.recon'), {})])
self._full_recon_path('container')), {})])
self.assertEqual(rv, {"container_updater_sweep": 18.476239919662476})
def test_get_updater_info_object(self):
@ -816,7 +824,7 @@ class TestReconSuccess(TestCase):
rv = self.app.get_updater_info('object')
self.assertEqual(self.fakecache.fakeout_calls,
[((['object_updater_sweep'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, {"object_updater_sweep": 0.79848217964172363})
def test_get_updater_info_unrecognized(self):
@ -831,7 +839,7 @@ class TestReconSuccess(TestCase):
rv = self.app.get_expirer_info('object')
self.assertEqual(self.fakecache.fakeout_calls,
[((['object_expiration_pass', 'expired_last_pass'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, from_cache_response)
def test_get_auditor_info_account(self):
@ -847,7 +855,7 @@ class TestReconSuccess(TestCase):
'account_auditor_pass_completed',
'account_audits_since',
'account_audits_failed'],
'/var/cache/swift/account.recon'), {})])
self._full_recon_path('account')), {})])
self.assertEqual(rv, {"account_auditor_pass_completed": 0.24,
"account_audits_failed": 0,
"account_audits_passed": 6,
@ -866,7 +874,7 @@ class TestReconSuccess(TestCase):
'container_auditor_pass_completed',
'container_audits_since',
'container_audits_failed'],
'/var/cache/swift/container.recon'), {})])
self._full_recon_path('container')), {})])
self.assertEqual(rv, {"container_auditor_pass_completed": 0.24,
"container_audits_failed": 0,
"container_audits_passed": 6,
@ -894,7 +902,7 @@ class TestReconSuccess(TestCase):
self.assertEqual(self.fakecache.fakeout_calls,
[((['object_auditor_stats_ALL',
'object_auditor_stats_ZBF'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, {
"object_auditor_stats_ALL": {
"audit_time": 115.14418768882751,
@ -941,7 +949,7 @@ class TestReconSuccess(TestCase):
self.assertEqual(self.fakecache.fakeout_calls,
[((['object_auditor_stats_ALL',
'object_auditor_stats_ZBF'],
'/var/cache/swift/object.recon'), {})])
self._full_recon_path('object')), {})])
self.assertEqual(rv, {
"object_auditor_stats_ALL": {
'disk1': {
@ -1130,7 +1138,8 @@ class TestReconSuccess(TestCase):
rv = self.app.get_driveaudit_error()
self.assertEqual(self.fakecache.fakeout_calls,
[((['drive_audit_errors'],
'/var/cache/swift/drive.recon'), {})])
self._full_recon_path(
None, recon_file=RECON_DRIVE_FILE)), {})])
self.assertEqual(rv, {'drive_audit_errors': 7})
def test_get_time(self):
@ -1218,7 +1227,7 @@ class TestReconSuccess(TestCase):
rv = self.app.get_sharding_info()
self.assertEqual(self.fakecache.fakeout_calls, [
((['sharding_stats', 'sharding_time', 'sharding_last'],
'/var/cache/swift/container.recon'), {})])
self._full_recon_path('container')), {})])
self.assertEqual(rv, from_cache_response)
def test_get_relinker_info(self):
@ -1299,7 +1308,8 @@ class TestReconSuccess(TestCase):
rv = self.app.get_relinker_info()
self.assertEqual(self.fakecache.fakeout_calls,
[((['devices', 'workers'],
'/var/cache/swift/relinker.recon'),
self._full_recon_path(
None, recon_file=RECON_RELINKER_FILE)),
{'ignore_missing': True})])
self.assertEqual(rv, from_cache_response)

View File

@ -43,7 +43,7 @@ class FakeDatabaseBroker(object):
class FakeDatabaseAuditor(DatabaseAuditor):
server_type = "test"
server_type = "container"
broker_class = FakeDatabaseBroker
def _audit(self, info, broker):
@ -136,7 +136,7 @@ class TestAuditor(unittest.TestCase):
self.assertEqual(test_auditor.failures, 1)
self.assertEqual(test_auditor.passes, 3)
def test_container_auditor(self):
def test_database_auditor(self):
conf = {}
test_auditor = FakeDatabaseAuditor(conf, logger=self.logger)
files = os.listdir(self.testdir)

View File

@ -2061,7 +2061,8 @@ class TestReplToNode(unittest.TestCase):
class ExampleReplicator(db_replicator.Replicator):
server_type = 'fake'
# We need to have a valid server_type
server_type = 'object'
brokerclass = ExampleBroker
datadir = 'fake'
default_port = 1000

View File

@ -0,0 +1,31 @@
# -*- coding:utf-8 -*-
# Copyright (c) 2010-2021 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import TestCase
from swift.common import recon
class TestCommonRecon(TestCase):
def test_server_type_to_recon_file(self):
# valid server types will come out as <server_type>.recon
for server_type in ('object', 'container', 'account', 'ACCount'):
self.assertEqual(recon.server_type_to_recon_file(server_type),
"%s.recon" % server_type.lower())
# other values will return a ValueError
for bad_server_type in ('obj', '', None, 'other', 'Account '):
self.assertRaises(ValueError,
recon.server_type_to_recon_file, bad_server_type)

View File

@ -37,6 +37,7 @@ from swift.common import utils
from swift.common.utils import (hash_path, mkdirs, normalize_timestamp,
storage_directory)
from swift.common import ring
from swift.common.recon import RECON_OBJECT_FILE
from swift.obj import diskfile, replicator as object_replicator
from swift.common.storage_policy import StoragePolicy, POLICIES
@ -385,7 +386,7 @@ class TestObjectReplicator(unittest.TestCase):
self.assertEqual((start + 1 + cycle) % 10,
replicator.replication_cycle)
recon_fname = os.path.join(self.recon_cache, "object.recon")
recon_fname = os.path.join(self.recon_cache, RECON_OBJECT_FILE)
with open(recon_fname) as cachefile:
recon = json.loads(cachefile.read())
self.assertEqual(1, recon.get('replication_time'))
@ -1454,7 +1455,7 @@ class TestObjectReplicator(unittest.TestCase):
# since we weren't operating on everything, but only a subset of
# storage policies, we didn't dump any recon stats.
self.assertFalse(os.path.exists(
os.path.join(self.recon_cache, 'object.recon')))
os.path.join(self.recon_cache, RECON_OBJECT_FILE)))
def test_delete_partition_ssync(self):
with mock.patch('swift.obj.replicator.http_connect',
@ -2229,7 +2230,7 @@ class TestMultiProcessReplicator(unittest.TestCase):
self.recon_cache = tempfile.mkdtemp()
rmtree(self.recon_cache, ignore_errors=1)
os.mkdir(self.recon_cache)
self.recon_file = os.path.join(self.recon_cache, 'object.recon')
self.recon_file = os.path.join(self.recon_cache, RECON_OBJECT_FILE)
bind_port = 6200