From 09370862cab5582c1fe422dfff7cd5ee0edcaac5 Mon Sep 17 00:00:00 2001 From: yuan-zhou Date: Tue, 19 Feb 2013 13:14:32 +0800 Subject: [PATCH] Adding speed limit options for DB auditor Fix bug 1129760 Without speed limit, DB auditor will likely consume high CPU% on storage node. That will highly impact the cluster's performance. This patch adds two options for account/container auditor: - containers_per_second: Maximum containers audited per second - accounts_per_second: Maximum accounts audited per second DocImpact Change-Id: I9faa506438185a83ca77db4906969328624d015f --- doc/manpages/account-server.conf.5 | 2 ++ doc/manpages/container-server.conf.5 | 2 ++ doc/source/deployment_guide.rst | 22 ++++++++++++++-------- etc/account-server.conf-sample | 1 + etc/container-server.conf-sample | 1 + swift/account/auditor.py | 7 ++++++- swift/container/auditor.py | 7 ++++++- 7 files changed, 32 insertions(+), 10 deletions(-) diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index 80677e0cf5..756fb88e91 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -210,6 +210,8 @@ Logging level. The default is INFO. Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 account per device per interval. The default is 1800 seconds. +.IP \fBaccounts_per_second\fR +Maximum accounts audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200. .RE diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 08575869a5..4067f12762 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -237,6 +237,8 @@ Logging level. The default is INFO. Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 container per device per interval. The default is 1800 seconds. +.IP \fBcontainers_per_second\fR +Maximum containers audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200. .RE diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index fa03243469..ab35c54224 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -424,14 +424,17 @@ account_suppression_time 60 Seconds to suppress updating an [container-auditor] -================== ================= ======================================= -Option Default Description ------------------- ----------------- --------------------------------------- -log_name container-auditor Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -interval 1800 Minimum time for a pass to take -================== ================= ======================================= +===================== ================= ======================================= +Option Default Description +--------------------- ----------------- --------------------------------------- +log_name container-auditor Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 1800 Minimum time for a pass to take +containers_per_second 200 Maximum containers audited per second. + Should be tuned according to individual + system specs. 0 is unlimited. +===================== ================= ======================================= ---------------------------- Account Server Configuration @@ -515,6 +518,9 @@ log_name account-auditor Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level interval 1800 Minimum time for a pass to take +accounts_per_second 200 Maximum accounts audited per second. + Should be tuned according to individual + system specs. 0 is unlimited. ==================== =============== ======================================= [account-reaper] diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 50244e5769..e1ab71458d 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -92,6 +92,7 @@ use = egg:swift#recon # interval = 1800 # log_facility = LOG_LOCAL0 # log_level = INFO +# accounts_per_second = 200 # recon_cache_path = /var/cache/swift [account-reaper] diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 7721b69be9..fb46a4869c 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -107,6 +107,7 @@ use = egg:swift#recon # log_address = /dev/log # Will audit, at most, 1 container per device per interval # interval = 1800 +# containers_per_second = 200 # recon_cache_path = /var/cache/swift [container-sync] diff --git a/swift/account/auditor.py b/swift/account/auditor.py index bb44b94da2..aa13acbf76 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -21,7 +21,7 @@ import swift.common.db from swift.account import server as account_server from swift.common.db import AccountBroker from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache + config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon from eventlet import Timeout @@ -38,6 +38,9 @@ class AccountAuditor(Daemon): self.interval = int(conf.get('interval', 1800)) self.account_passes = 0 self.account_failures = 0 + self.accounts_running_time = 0 + self.max_accounts_per_second = \ + float(conf.get('accounts_per_second', 200)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', @@ -67,6 +70,8 @@ class AccountAuditor(Daemon): reported = time.time() self.account_passes = 0 self.account_failures = 0 + self.accounts_running_time = ratelimit_sleep( + self.accounts_running_time, self.max_accounts_per_second) return reported def run_forever(self, *args, **kwargs): diff --git a/swift/container/auditor.py b/swift/container/auditor.py index 437163589b..4fc112c495 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -23,7 +23,7 @@ import swift.common.db from swift.container import server as container_server from swift.common.db import ContainerBroker from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache + config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon @@ -38,6 +38,9 @@ class ContainerAuditor(Daemon): self.interval = int(conf.get('interval', 1800)) self.container_passes = 0 self.container_failures = 0 + self.containers_running_time = 0 + self.max_containers_per_second = \ + float(conf.get('containers_per_second', 200)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', @@ -66,6 +69,8 @@ class ContainerAuditor(Daemon): reported = time.time() self.container_passes = 0 self.container_failures = 0 + self.containers_running_time = ratelimit_sleep( + self.containers_running_time, self.max_containers_per_second) return reported def run_forever(self, *args, **kwargs):