openstack
/
swift
Code Issues Proposed changes

Adding speed limit options for DB auditor 

Fix bug 1129760

Without speed limit, DB auditor will likely consume high CPU% on
storage node. That will highly impact the cluster's performance.

This patch adds two options for account/container auditor:
 - containers_per_second: Maximum containers audited per second
 - accounts_per_second: Maximum accounts audited per second

DocImpact

Change-Id: I9faa506438185a83ca77db4906969328624d015f
This commit is contained in:
yuan-zhou 2013-02-19 13:14:32 +08:00
parent e88ff34685
commit 09370862ca
7 changed files with 32 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/manpages/account-server.conf.5
View File

@ -210,6 +210,8 @@ Logging level. The default is INFO.
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Will audit, at most, 1 account per device per interval. The default is 1800 seconds.
.IP \fBaccounts_per_second\fR
Maximum accounts audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200.
.RE

2
doc/manpages/container-server.conf.5
View File

@ -237,6 +237,8 @@ Logging level. The default is INFO.
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Will audit, at most, 1 container per device per interval. The default is 1800 seconds.
.IP \fBcontainers_per_second\fR
Maximum containers audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200.
.RE

22
doc/source/deployment_guide.rst
View File

@ -424,14 +424,17 @@ account_suppression_time 60 Seconds to suppress updating an
[container-auditor]
================== ================= =======================================
Option Default Description
------------------ ----------------- ---------------------------------------
log_name container-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
================== ================= =======================================
===================== ================= =======================================
Option Default Description
--------------------- ----------------- ---------------------------------------
log_name container-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
containers_per_second 200 Maximum containers audited per second.
Should be tuned according to individual
system specs. 0 is unlimited.
===================== ================= =======================================
----------------------------
Account Server Configuration
@ -515,6 +518,9 @@ log_name account-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
accounts_per_second 200 Maximum accounts audited per second.
Should be tuned according to individual
system specs. 0 is unlimited.
==================== =============== =======================================
[account-reaper]

1
etc/account-server.conf-sample
View File

@ -92,6 +92,7 @@ use = egg:swift#recon
# interval = 1800
# log_facility = LOG_LOCAL0
# log_level = INFO
# accounts_per_second = 200
# recon_cache_path = /var/cache/swift
[account-reaper]

1
etc/container-server.conf-sample
View File

@ -107,6 +107,7 @@ use = egg:swift#recon
# log_address = /dev/log
# Will audit, at most, 1 container per device per interval
# interval = 1800
# containers_per_second = 200
# recon_cache_path = /var/cache/swift
[container-sync]

7
swift/account/auditor.py
View File

@ -21,7 +21,7 @@ import swift.common.db
from swift.account import server as account_server
from swift.common.db import AccountBroker
from swift.common.utils import get_logger, audit_location_generator, \
config_true_value, dump_recon_cache
config_true_value, dump_recon_cache, ratelimit_sleep
from swift.common.daemon import Daemon
from eventlet import Timeout
@ -38,6 +38,9 @@ class AccountAuditor(Daemon):
self.interval = int(conf.get('interval', 1800))
self.account_passes = 0
self.account_failures = 0
self.accounts_running_time = 0
self.max_accounts_per_second = \
float(conf.get('accounts_per_second', 200))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.recon_cache_path = conf.get('recon_cache_path',
@ -67,6 +70,8 @@ class AccountAuditor(Daemon):
reported = time.time()
self.account_passes = 0
self.account_failures = 0
self.accounts_running_time = ratelimit_sleep(
self.accounts_running_time, self.max_accounts_per_second)
return reported
def run_forever(self, *args, **kwargs):

7
swift/container/auditor.py
View File

@ -23,7 +23,7 @@ import swift.common.db
from swift.container import server as container_server
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, audit_location_generator, \
config_true_value, dump_recon_cache
config_true_value, dump_recon_cache, ratelimit_sleep
from swift.common.daemon import Daemon
@ -38,6 +38,9 @@ class ContainerAuditor(Daemon):
self.interval = int(conf.get('interval', 1800))
self.container_passes = 0
self.container_failures = 0
self.containers_running_time = 0
self.max_containers_per_second = \
float(conf.get('containers_per_second', 200))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.recon_cache_path = conf.get('recon_cache_path',
@ -66,6 +69,8 @@ class ContainerAuditor(Daemon):
reported = time.time()
self.container_passes = 0
self.container_failures = 0
self.containers_running_time = ratelimit_sleep(
self.containers_running_time, self.max_containers_per_second)
return reported
def run_forever(self, *args, **kwargs):