openstack/swift
Code Issues Proposed changes

Initial commit of middleware refactor

Browse Source
This commit is contained in:
Chuck Thier 2010-08-20 00:42:38 +00:00
parent dab46ea017
commit 2c596c0a0f
52 changed files with 535 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bin/swift-account-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-auditor'):
auditor_conf = dict(c.items('account-auditor'))
conf = dict(c.items('account-auditor'))
else:
print "Unable to find account-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'account-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = AccountAuditor(server_conf, auditor_conf)
auditor = AccountAuditor(conf)
if once:
auditor.audit_once()
else:

9
bin/swift-account-reaper
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-reaper'):
reaper_conf = dict(c.items('account-reaper'))
conf = dict(c.items('account-reaper'))
else:
print "Unable to find account-reaper config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(reaper_conf, 'account-reaper')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
reaper = AccountReaper(server_conf, reaper_conf)
reaper = AccountReaper(conf)
if once:
reaper.reap_once()
else:

9
bin/swift-account-replicator
View File

@ -41,17 +41,16 @@ if __name__ == '__main__':
sys.exit(1)
once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('account-server'))
if c.has_section('account-replicator'):
replicator_conf = dict(c.items('account-replicator'))
conf = dict(c.items('account-replicator'))
else:
print "Unable to find account-replicator config section in %s." % \
args[0]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]:
AccountReplicator(server_conf, replicator_conf).replicate_once()
AccountReplicator(conf).replicate_once()
else:
AccountReplicator(server_conf, replicator_conf).replicate_forever()
AccountReplicator(conf).replicate_forever()

11
bin/swift-account-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.account.server import AccountController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('account-server'))
run_wsgi(AccountController, conf, default_port=6002)
if len(sys.argv) != 2:
print "Usage: %s CONFIG_FILE"
run_wsgi(sys.argv[1], default_port=6002)

11
bin/swift-auth-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.auth.server import AuthController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('auth-server'))
run_wsgi(AuthController, conf, default_port=11000)
if len(sys.argv) != 2:
print "Usage: %s CONFIG_FILE"
run_wsgi(sys.argv[1], default_port=11000)

9
bin/swift-container-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-auditor'):
auditor_conf = dict(c.items('container-auditor'))
conf = dict(c.items('container-auditor'))
else:
print "Unable to find container-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'container-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(onf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = ContainerAuditor(server_conf, auditor_conf)
auditor = ContainerAuditor(conf)
if once:
auditor.audit_once()
else:

9
bin/swift-container-replicator
View File

@ -41,17 +41,16 @@ if __name__ == '__main__':
sys.exit(1)
once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('container-server'))
if c.has_section('container-replicator'):
replicator_conf = dict(c.items('container-replicator'))
conf = dict(c.items('container-replicator'))
else:
print "Unable to find container-replicator config section in %s." % \
args[0]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]:
ContainerReplicator(server_conf, replicator_conf).replicate_once()
ContainerReplicator(conf).replicate_once()
else:
ContainerReplicator(server_conf, replicator_conf).replicate_forever()
ContainerReplicator(conf).replicate_forever()

11
bin/swift-container-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.container.server import ContainerController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('container-server'))
run_wsgi(ContainerController, conf, default_port=6001)
if len(sys.argv) != 2:
print "Usage: %s CONFIG_FILE"
run_wsgi(sys.argv[1], default_port=6001)

7
bin/swift-container-updater
View File

@ -34,15 +34,14 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-updater'):
updater_conf = dict(c.items('container-updater'))
conf = dict(c.items('container-updater'))
else:
print "Unable to find container-updater config section in %s." % \
sys.argv[1]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -56,7 +55,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
updater = ContainerUpdater(server_conf, updater_conf)
updater = ContainerUpdater(conf)
if once:
updater.update_once_single_threaded()
else:

9
bin/swift-object-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-auditor'):
auditor_conf = dict(c.items('object-auditor'))
conf = dict(c.items('object-auditor'))
else:
print "Unable to find object-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'object-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = ObjectAuditor(server_conf, auditor_conf)
auditor = ObjectAuditor(conf)
if once:
auditor.audit_once()
else:

39
bin/swift-object-replicator
View File

@ -27,38 +27,23 @@ from swift.common.utils import get_logger, drop_privileges, LoggerFileObject
TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes'))
def read_configs(conf_file):
c = ConfigParser()
if not c.read(conf_file):
print "Unable to read config file: %s" % conf_file
sys.exit(1)
conf = dict(c.items('object-server'))
repl_conf = dict(c.items('object-replicator'))
if not repl_conf:
sys.exit()
conf['replication_concurrency'] = repl_conf.get('concurrency',1)
conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no')
conf['daemonize'] = repl_conf.get('daemonize', 'yes')
conf['run_pause'] = repl_conf.get('run_pause', '30')
conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1')
conf['log_level'] = repl_conf.get('log_level', 'INFO')
conf['timeout'] = repl_conf.get('timeout', '5')
conf['stats_interval'] = repl_conf.get('stats_interval', '3600')
conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400))
return conf
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: object-replicator CONFIG_FILE [once]"
sys.exit()
try:
conf = read_configs(sys.argv[1])
except:
print "Problem reading the config. Aborting object replication."
sys.exit()
c = ConfigParser()
if not c.read(conf_file):
print "Unable to read config file: %s" % conf_file
sys.exit(1)
if c.has_section('object-replicator'):
conf = dict(c.items('object-replicator'))
else:
print "Unable to find object-replicator config section in %s" % \
sys.argv[1]
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
logger = get_logger(conf, 'object-replicator')
logger = get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)

12
bin/swift-object-server
View File

@ -14,17 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.obj.server import ObjectController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('object-server'))
run_wsgi(ObjectController, conf, default_port=6000)
if len(sys.argv) != 2:
print "Usage: %s CONFIG_FILE"
run_wsgi(sys.argv[1], default_port=6000)

7
bin/swift-object-updater
View File

@ -34,15 +34,14 @@ if __name__ == '__main__':
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-updater'):
updater_conf = dict(c.items('object-updater'))
conf = dict(c.items('object-updater'))
else:
print "Unable to find object-updater config section in %s." % \
sys.argv[1]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -56,7 +55,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
updater = ObjectUpdater(server_conf, updater_conf)
updater = ObjectUpdater(conf)
if once:
updater.update_once_single_threaded()
else:

32
bin/swift-proxy-server
View File

@ -14,37 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import os
import sys
from swift.common.wsgi import run_wsgi
from swift.common.auth import DevAuthMiddleware
from swift.common.memcached import MemcacheRing
from swift.common.utils import get_logger
from swift.proxy.server import Application
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('proxy-server'))
if c.has_section('auth-server'):
auth_conf = dict(c.items('auth-server'))
else:
auth_conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
m, c = auth_conf.get('class',
'swift.common.auth.DevAuthMiddleware').rsplit('.', 1)
m = __import__(m, fromlist=[c])
authware = m.__dict__[c]
memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
logger = get_logger(conf, 'proxy')
app = Application(conf, memcache, logger)
# Wrap the app with auth
app = authware(app, auth_conf, memcache, logger)
run_wsgi(app, conf, logger=logger, default_port=80)
if len(sys.argv) != 2:
print "Usage: %s CONFIG_FILE"
run_wsgi(sys.argv[1], default_port=80)

23
etc/account-server.conf-sample
View File

@ -1,15 +1,23 @@
[account-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
log_name = account
# log_facility = LOG_LOCAL0
# log_level = INFO
# bind_ip = 0.0.0.0
# bind_port = 6002
# workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
log_name = account-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# per_diff = 1000
@ -26,6 +34,7 @@
# reclaim_age = 86400
[account-stats]
log_name = account-stats
# cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31
# container_name = account_stats
# proxy_server_conf = /etc/swift/proxy-server.conf
@ -33,6 +42,7 @@
# log_level = INFO
[account-auditor]
log_name = account-auditor
# Will audit, at most, 1 account per device per interval
# interval = 1800
# Maximum containers randomly picked for a given account audit
@ -43,6 +53,7 @@
# log_level = INFO
[account-reaper]
log_name = account-reaper
# concurrency = 25
# interval = 3600
# node_timeout = 10

17
etc/auth-server.conf-sample
View File

@ -1,15 +1,22 @@
[auth-server]
# swift_dir = /etc/swift
[DEFAULT]
log_name = auth
# bind_ip = 0.0.0.0
# bind_port = 11000
# log_facility = LOG_LOCAL0
# log_level = INFO
# workers = 1
# user = swift
# swift_dir = /etc/swift
# cert_file = Default is no cert; format is path like /etc/swift/auth.crt
# key_file = Default is no key; format is path like /etc/swift/auth.key
[pipeline:main]
pipeline = auth-server
[app:auth-server]
use = egg:swift#auth
# reseller_prefix = AUTH
# default_cluster_url = http://127.0.0.1:9000/v1
# token_life = 86400
# log_headers = False
# cert_file = Default is no cert; format is path like /etc/swift/auth.crt
# key_file = Default is no key; format is path like /etc/swift/auth.key
# node_timeout = 10
user = swift

28
etc/container-server.conf-sample
View File

@ -1,19 +1,25 @@
[container-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
log_name = container
# log_facility = LOG_LOCAL0
# log_level = INFO
# bind_ip = 0.0.0.0
# bind_port = 6001
# workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
# node_timeout = 3
# conn_timeout = 0.5
[container-replicator]
# log_facility = LOG_LOCAL0
# log_level = INFO
log_name = container-replicator
# per_diff = 1000
# concurrency = 8
# run_pause = 30
@ -23,21 +29,19 @@
# reclaim_age = 604800
[container-updater]
log_name = container-updater
# interval = 300
# concurrency = 4
# node_timeout = 3
# conn_timeout = 0.5
# slowdown will sleep that amount between containers
# slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[container-auditor]
log_name = container-auditor
# Will audit, at most, 1 container per device per interval
# interval = 1800
# Maximum objects randomly picked for a given container audit
# max_object_count = 100
# node_timeout = 10
# conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

34
etc/object-server.conf-sample
View File

@ -1,24 +1,30 @@
[object-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
log_name = object
# log_facility = LOG_LOCAL0
# log_level = INFO
# bind_ip = 0.0.0.0
# bind_port = 6000
# workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_requests = True
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
# log_requests = True
# node_timeout = 3
# conn_timeout = 0.5
# network_chunk_size = 8192
# disk_chunk_size = 32768
# network_chunk_size = 65536
# disk_chunk_size = 65536
# max_upload_time = 86400
# slow = 1
[object-replicator]
# log_facility = LOG_LOCAL0
# log_level = INFO
log_name = object-replicator
# daemonize = on
# run_pause = 30
# concurrency = 1
@ -28,19 +34,17 @@
# reclaim_age = 604800
[object-updater]
log_name = object-updater
# interval = 300
# concurrency = 1
# node_timeout = 10
# conn_timeout = 0.5
# slowdown will sleep that amount between objects
# slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[object-auditor]
log_name = object-auditor
# Will audit, at most, 1 object per device per interval
# interval = 1800
# node_timeout = 10
# conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

38
etc/proxy-server.conf-sample
View File

@ -1,21 +1,26 @@
[proxy-server]
# bind_ip = 0.0.0.0
# bind_port = 80
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
# swift_dir = /etc/swift
[DEFAULT]
log_name = proxy
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_headers = False
# bind_ip = 0.0.0.0
# bind_port = 80
# workers = 1
# user = swift
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
[pipeline:main]
pipeline = healthcheck cache auth proxy
[app:proxy]
use = egg:swift#proxy
filter-with = auth
# swift_dir = /etc/swift
# log_headers = False
# recheck_account_existence = 60
# recheck_container_existence = 60
# object_chunk_size = 8192
# client_chunk_size = 8192
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211
# node_timeout = 10
# client_timeout = 60
# conn_timeout = 0.5
@ -31,8 +36,17 @@
# rate_limit_account_whitelist = acct1,acct2,etc
# rate_limit_account_blacklist = acct3,acct4,etc
# [auth-server]
# class = swift.common.auth.DevAuthMiddleware
[filter:auth]
use = egg:swift#auth
# ip = 127.0.0.1
# port = 11000
# node_timeout = 10
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#cache
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211

51
setup.py
View File

@ -55,21 +55,38 @@ setup(
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.6',
'Environment :: No Input/Output (Daemon)',
],
],
install_requires=[], # removed for better compat
scripts=['bin/st', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-create-account',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-auditor',
'bin/swift-container-replicator',
'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-info',
'bin/swift-object-replicator',
'bin/swift-object-server',
'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-stats-report'])
scripts=[
'bin/st', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-create-account',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-auditor',
'bin/swift-container-replicator',
'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-info',
'bin/swift-object-replicator',
'bin/swift-object-server',
'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-stats-report'
],
entry_points={
'paste.app_factory' : [
'proxy=swift.proxy.server:app_factory',
'object=swift.obj.server:app_factory',
'container=swift.container.server:app_factory',
'account=swift.account.server:app_factory',
'auth=swift.auth.server:app_factory',
],
'paste.filter_factory' : [
'auth=swift.common.middleware.auth:filter_factory',
'healthcheck=swift.common.middleware.healthcheck:filter_factory',
'cache=swift.common.middleware.cache:filter_factory',
],
},
)

18
swift/account/auditor.py
View File

@ -35,19 +35,19 @@ class AuditException(Exception):
class AccountAuditor(object):
"""Audit accounts."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'account-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf, 'account-auditor')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_container_count = \
int(auditor_conf.get('max_container_count', 100))
int(conf.get('max_container_count', 100))
self.container_passes = 0
self.container_failures = 0
self.container_errors = 0

20
swift/account/reaper.py
View File

@ -50,25 +50,23 @@ class AccountReaper(object):
configuration parameters.
"""
log_name = 'account-reaper'
def __init__(self, server_conf, reaper_conf):
self.logger = get_logger(reaper_conf, self.log_name)
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(reaper_conf.get('interval', 3600))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 3600))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.account_ring = None
self.container_ring = None
self.object_ring = None
self.node_timeout = int(reaper_conf.get('node_timeout', 10))
self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.myips = whataremyips()
self.concurrency = int(reaper_conf.get('concurrency', 25))
self.concurrency = int(conf.get('concurrency', 25))
self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency)

16
swift/account/server.py
View File

@ -31,10 +31,9 @@ from xml.sax import saxutils
from swift.common.db import AccountBroker
from swift.common.utils import get_param, split_path, storage_directory, \
hash_path
hash_path, get_logger
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable
from swift.common.healthcheck import healthcheck
from swift.common.db_replicator import ReplicatorRpc
@ -43,10 +42,9 @@ DATADIR = 'accounts'
class AccountController(object):
"""WSGI controller for the account server."""
log_name = 'account'
def __init__(self, conf):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -253,9 +251,7 @@ class AccountController(object):
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -288,3 +284,9 @@ class AccountController(object):
else:
self.logger.info(log_message)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI account server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return AccountController(conf)

9
swift/auth/server.py
View File

@ -89,10 +89,8 @@ class AuthController(object):
configuration parameters.
"""
log_name = 'auth'
def __init__(self, conf, ring=None):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.default_cluster_url = \
conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1')
@ -500,3 +498,8 @@ class AuthController(object):
def __call__(self, env, start_response):
""" Used by the eventlet.wsgi.server """
return self.handleREST(env, start_response)
def app_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
return AuthController(conf)

26
swift/common/db_replicator.py
View File

@ -89,28 +89,28 @@ class Replicator(object):
Implements the logic for directing db replication.
"""
def __init__(self, server_conf, replicator_conf):
def __init__(self, conf):
self.logger = \
get_logger(replicator_conf, '%s-replicator' % self.server_type)
get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(self.logger)
self.root = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.port = int(server_conf.get('bind_port', self.default_port))
concurrency = int(replicator_conf.get('concurrency', 8))
self.port = int(conf.get('bind_port', self.default_port))
concurrency = int(conf.get('concurrency', 8))
self.cpool = GreenPool(size=concurrency)
swift_dir = server_conf.get('swift_dir', '/etc/swift')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file))
self.per_diff = int(replicator_conf.get('per_diff', 1000))
self.run_pause = int(replicator_conf.get('run_pause', 30))
self.vm_test_mode = replicator_conf.get(
self.per_diff = int(conf.get('per_diff', 1000))
self.run_pause = int(conf.get('run_pause', 30))
self.vm_test_mode = conf.get(
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.node_timeout = int(replicator_conf.get('node_timeout', 10))
self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5))
self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
self._zero_stats()
def _zero_stats(self):

29
swift/common/healthcheck.py
View File

@ -1,29 +0,0 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Response
class HealthCheckController(object):
"""Basic controller used for monitoring."""
def __init__(self, *args, **kwargs):
pass
@classmethod
def GET(self, req):
return Response(request=req, body="OK", content_type="text/plain")
healthcheck = HealthCheckController.GET

0
swift/common/middleware/__init__.py Normal file
View File

45
swift/common/auth.py → swift/common/middleware/auth.py
View File

@ -21,6 +21,8 @@ from eventlet.timeout import Timeout
from swift.common.utils import split_path
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import get_logger, cache_from_env
from swift.common.memcached import MemcacheRing
class DevAuthMiddleware(object):
@ -28,33 +30,37 @@ class DevAuthMiddleware(object):
Auth Middleware that uses the dev auth server
"""
def __init__(self, app, conf, memcache_client, logger):
def __init__(self, app, conf, memcache_client=None, logger=None):
self.app = app
self.memcache_client = memcache_client
self.logger = logger
if logger is None:
self.logger = get_logger(conf)
else:
self.logger = logger
self.conf = conf
self.auth_host = conf.get('ip', '127.0.0.1')
self.auth_port = int(conf.get('port', 11000))
self.timeout = int(conf.get('node_timeout', 10))
def __call__(self, env, start_response):
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
req = Request(env)
if req.path != '/healthcheck':
if 'x-storage-token' in req.headers and \
'x-auth-token' not in req.headers:
req.headers['x-auth-token'] = req.headers['x-storage-token']
if 'x-storage-token' in req.headers and \
'x-auth-token' not in req.headers:
req.headers['x-auth-token'] = req.headers['x-storage-token']
try:
version, account, container, obj = split_path(req.path, 1, 4, True)
if account is None:
return HTTPPreconditionFailed(request=req, body='Bad URL')(
env, start_response)
if not req.headers.get('x-auth-token'):
return HTTPPreconditionFailed(request=req,
body='Missing Auth Token')(env, start_response)
if account is None:
return HTTPPreconditionFailed(
request=req, body='Bad URL')(env, start_response)
if not self.auth(account, req.headers['x-auth-token']):
return HTTPUnauthorized(request=req)(env, start_response)
except ValueError, e:
version = account = container = obj = None
if account is None:
return HTTPPreconditionFailed(request=req, body='Bad URL')(
env, start_response)
if not req.headers.get('x-auth-token'):
return HTTPPreconditionFailed(request=req,
body='Missing Auth Token')(env, start_response)
if not self.auth(account, req.headers['x-auth-token']):
return HTTPUnauthorized(request=req)(env, start_response)
# If we get here, then things should be good.
return self.app(env, start_response)
@ -95,3 +101,8 @@ class DevAuthMiddleware(object):
val = (now, validated)
self.memcache_client.set(key, val, timeout=validated)
return True
def filter_factory(global_conf, **local_conf):
def auth_filter(app):
return DevAuthMiddleware(app, local_conf)
return auth_filter

34
swift/common/middleware/cache.py Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.common.memcached import MemcacheRing
class CacheMiddleware(object):
"""
Caching middleware that manages caching in swift.
"""
def __init__(self, app, conf):
self.app = app
self.memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
def __call__(self, env, start_response):
env['swift.cache'] = self.memcache
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def cache_filter(app):
return CacheMiddleware(app, local_conf)
return cache_filter

43
swift/common/middleware/healthcheck.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Request, Response
class HealthCheckMiddleware(object):
"""
Healthcheck middleware used for monitoring.
If the path is /healthcheck, it will respond with "OK" in the body
"""
def __init__(self, app, *args, **kwargs):
self.app = app
def GET(self, req):
"""Returns a 200 response with "OK" in the body."""
return Response(request=req, body="OK", content_type="text/plain")
def __call__(self, env, start_response):
req = Request(env)
if req.path == '/healthcheck':
return self.GET(req)(env, start_response)
else:
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def healthcheck_filter(app):
return HealthCheckMiddleware(app)
return healthcheck_filter

13
swift/common/utils.py
View File

@ -323,7 +323,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name):
def get_logger(conf, name=None):
"""
Get the current system logger using config settings.
@ -342,6 +342,8 @@ def get_logger(conf, name):
if conf is None:
root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name)
if name is None:
name = conf.get('log_name', 'swift')
get_logger.handler = SysLogHandler(address='/dev/log',
facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'),
SysLogHandler.LOG_LOCAL0))
@ -513,3 +515,12 @@ def unlink_older_than(path, mtime):
os.unlink(fpath)
except OSError:
pass
def item_from_env(env, item_name):
item = env.get(item_name, None)
if item is None:
logging.error("ERROR: %s could not be found in env!" % item_name)
return item
def cache_from_env(env):
return item_from_env(env, 'swift.cache')

18
swift/common/wsgi.py
View File

@ -24,6 +24,7 @@ import mimetools
import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen
from paste.deploy import loadapp, appconfig
# Hook to ensure connection resets don't blow up our servers.
# Remove with next release of Eventlet that has it in the set already.
@ -58,18 +59,24 @@ def monkey_patch_mimetools():
# We might be able to pull pieces of this out to test, but right now it seems
# like more work than it's worth.
def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover
def run_wsgi(conf_file, *args, **kwargs): # pragma: no cover
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param app: WSGI callable
:param conf: Configuration dictionary
:param conf_file: Path to paste.deploy style configuration file
"""
try:
app = loadapp('config:%s' % conf_file)
conf = appconfig('config:%s' % conf_file)
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs:
logger = kwargs['logger']
else:
logger = get_logger(conf, app.log_name)
logger = get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
@ -103,9 +110,6 @@ def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1'))
drop_privileges(conf.get('user', 'swift'))
if isinstance(app, type):
# Instantiate app if it hasn't been already
app = app(conf, *args)
def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"

18
swift/container/auditor.py
View File

@ -36,20 +36,20 @@ class AuditException(Exception):
class ContainerAuditor(object):
"""Audit containers."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'container-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.object_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.max_object_count = int(auditor_conf.get('max_object_count', 100))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_object_count = int(conf.get('max_object_count', 100))
self.account_passes = 0
self.account_failures = 0
self.account_errors = 0

17
swift/container/server.py
View File

@ -31,11 +31,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, get_param, hash_path, \
storage_directory, split_path
storage_directory, split_path, get_logger
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable
from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import healthcheck
from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc
@ -45,10 +44,8 @@ DATADIR = 'containers'
class ContainerController(object):
"""WSGI Controller for the container server."""
log_name = 'container'
def __init__(self, conf):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -347,9 +344,7 @@ class ContainerController(object):
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -378,3 +373,9 @@ class ContainerController(object):
else:
self.logger.info(log_message)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI container server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ContainerController(conf)

20
swift/container/updater.py
View File

@ -33,19 +33,19 @@ from swift.common.utils import get_logger, whataremyips
class ContainerUpdater(object):
"""Update container information in account listings."""
def __init__(self, server_conf, updater_conf):
self.logger = get_logger(updater_conf, 'container-updater')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf, 'container-updater')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None
self.concurrency = int(updater_conf.get('concurrency', 4))
self.slowdown = float(updater_conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 3))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5))
self.concurrency = int(conf.get('concurrency', 4))
self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.no_changes = 0
self.successes = 0
self.failures = 0

16
swift/obj/auditor.py
View File

@ -33,17 +33,17 @@ from swift.common.exceptions import AuditException
class ObjectAuditor(object):
"""Audit objects."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'object-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.passes = 0
self.quarantines = 0
self.errors = 0

2
swift/obj/replicator.py
View File

@ -215,7 +215,7 @@ class ObjectReplicator(object):
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('replication_concurrency', 1))
self.concurrency = int(conf.get('concurrency', 1))
self.timeout = conf['timeout']
self.stats_interval = int(conf['stats_interval'])
self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz'))

19
swift/obj/server.py
View File

@ -36,9 +36,8 @@ from xattr import getxattr, setxattr
from eventlet import sleep, Timeout
from swift.common.utils import mkdirs, normalize_timestamp, \
storage_directory, hash_path, get_logger, renamer, fallocate, \
split_path, drop_buffer_cache
from swift.common.healthcheck import healthcheck
storage_directory, hash_path, renamer, fallocate, \
split_path, drop_buffer_cache, get_logger
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, check_mount, \
check_float, check_xml_encodable
@ -242,8 +241,6 @@ class DiskFile(object):
class ObjectController(object):
"""Implements the WSGI application for the Swift Object Server."""
log_name = 'object'
def __init__(self, conf):
"""
Creates a new WSGI application for the Swift Object Server. An
@ -251,7 +248,7 @@ class ObjectController(object):
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -560,9 +557,7 @@ class ObjectController(object):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -596,3 +591,9 @@ class ObjectController(object):
if slow > 0:
sleep(slow)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI object server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ObjectController(conf)

20
swift/obj/updater.py
View File

@ -32,19 +32,19 @@ from swift.obj.server import ASYNCDIR
class ObjectUpdater(object):
"""Update object information in container listings."""
def __init__(self, server_conf, updater_conf):
self.logger = get_logger(updater_conf, 'object-updater')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.concurrency = int(updater_conf.get('concurrency', 1))
self.slowdown = float(updater_conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 10))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5))
self.concurrency = int(conf.get('concurrency', 1))
self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.successes = 0
self.failures = 0

33
swift/proxy/server.py
View File

@ -31,9 +31,9 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \
from webob import Request, Response
from swift.common.ring import Ring
from swift.common.utils import get_logger, normalize_timestamp, split_path
from swift.common.utils import get_logger, normalize_timestamp, split_path, \
cache_from_env
from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import HealthCheckController
from swift.common.constraints import check_object_creation, check_metadata, \
MAX_FILE_SIZE, check_xml_encodable
from swift.common.exceptions import ChunkReadTimeout, \
@ -47,7 +47,7 @@ def update_headers(response, headers):
Helper function to update headers in the response.
:param response: webob.Response object
:param headers: dictionary headers
:parm headers: dictionary headers
"""
if hasattr(headers, 'items'):
headers = headers.items()
@ -930,14 +930,12 @@ class AccountController(Controller):
class BaseApplication(object):
"""Base WSGI application for the proxy server"""
log_name = 'base_application'
def __init__(self, conf, memcache, logger=None, account_ring=None,
def __init__(self, conf, memcache=None, logger=None, account_ring=None,
container_ring=None, object_ring=None):
if logger:
self.logger = logger
if logger is None:
self.logger = get_logger(conf)
else:
self.logger = get_logger(conf, self.log_name)
self.logger = logger
if conf is None:
conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
@ -991,8 +989,6 @@ class BaseApplication(object):
return ContainerController, d
elif account and not container and not obj:
return AccountController, d
elif version and version == 'healthcheck':
return HealthCheckController, d
return None, d
def __call__(self, env, start_response):
@ -1004,6 +1000,8 @@ class BaseApplication(object):
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
req = self.update_request(Request(env))
if 'eventlet.posthooks' in env:
env['eventlet.posthooks'].append(
@ -1048,13 +1046,6 @@ class BaseApplication(object):
controller, path_parts = self.get_controller(req.path)
except ValueError:
return HTTPNotFound(request=req)
if controller == HealthCheckController:
controller = controller(self, **path_parts)
controller.trans_id = req.headers.get('x-cf-trans-id', '-')
if req.method == 'GET':
return controller.GET(req)
return HTTPMethodNotAllowed(request=req)
if not check_xml_encodable(req.path_info):
return HTTPPreconditionFailed(request=req, body='Invalid UTF8')
if not controller:
@ -1086,8 +1077,6 @@ class BaseApplication(object):
class Application(BaseApplication):
"""WSGI application for the proxy server."""
log_name = 'proxy'
def handle_request(self, req):
"""
Wraps the BaseApplication.handle_request and logs the request.
@ -1181,3 +1170,7 @@ class Application(BaseApplication):
return Response(status='498 Rate Limited',
body='Slow down', request=req)
return None
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
return Application(local_conf)

27
test/unit/account/test_server.py
View File

@ -795,33 +795,6 @@ class TestAccountController(unittest.TestCase):
listing.append(node2.firstChild.nodeValue)
self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2'])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self):
inbuf = StringIO()
errbuf = StringIO()

2
test/unit/auth/test_server.py
View File

@ -75,7 +75,7 @@ class TestAuthServer(unittest.TestCase):
'auth_server')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
self.conf = {'swift_dir': self.testdir}
self.conf = {'swift_dir': self.testdir, 'log_name': 'auth'}
self.controller = auth_server.AuthController(self.conf, FakeRing())
def tearDown(self):

0
test/unit/common/middleware/__init__.py Normal file
View File

2
test/unit/common/test_auth.py → test/unit/common/middleware/test_auth.py
View File

@ -23,7 +23,7 @@ from contextlib import contextmanager
import eventlet
from webob import Request
from swift.common import auth
from swift.common.middleware import auth
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

42
test/unit/common/middleware/test_cache.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from webob import Request
from swift.common.middleware import cache
from swift.common.memcached import MemcacheRing
class FakeApp(object):
def __call__(self, env, start_response):
return env
def start_response(*args):
pass
class TestCacheMiddleware(unittest.TestCase):
def setUp(self):
self.app = cache.CacheMiddleware(FakeApp(), {})
def test_cache_middleware(self):
req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertTrue('swift.cache' in resp)
self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing))
if __name__ == '__main__':
unittest.main()

24
test/unit/common/test_healthcheck.py → test/unit/common/middleware/test_healthcheck.py
View File

@ -17,17 +17,29 @@ import unittest
from webob import Request
from swift.common import healthcheck
from swift.common.middleware import healthcheck
class FakeApp(object):
def __call__(self, env, start_response):
return "FAKE APP"
def start_response(*args):
pass
class TestHealthCheck(unittest.TestCase):
def test_healthcheck(self):
controller = healthcheck.HealthCheckController()
req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'})
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
def setUp(self):
self.app = healthcheck.HealthCheckMiddleware(FakeApp())
def test_healthcheck(self):
req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['OK'])
def test_healtcheck_pass(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
if __name__ == '__main__':
unittest.main()

16
test/unit/common/test_db_replicator.py
View File

@ -139,7 +139,7 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals(conn.post(1, 2, 3), None)
def test_rsync_file(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
with _mock_process(-1):
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
self.assertEquals(False,
@ -150,13 +150,13 @@ class TestDBReplicator(unittest.TestCase):
replicator._rsync_file('/some/file', 'remote:/some/file'))
def test_rsync_db(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._rsync_file = lambda *args: True
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd')
def test_in_sync(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
self.assertEquals(replicator._in_sync(
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
@ -171,16 +171,16 @@ class TestDBReplicator(unittest.TestCase):
FakeBroker(), -1)), False)
def test_replicate_once(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator.replicate_once()
def test_usync(self):
fake_http = PostReplHttp()
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')
def test_repl_to_node(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000}
fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b',
'created_at': 100, 'put_timestamp': 0,
@ -192,13 +192,13 @@ class TestDBReplicator(unittest.TestCase):
def test_stats(self):
# I'm not sure how to test that this logs the right thing,
# but we can at least make sure it gets covered.
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._zero_stats()
replicator._report_stats()
def test_replicate_object(self):
db_replicator.lock_parent_directory = lock_parent_directory
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._replicate_object('0', 'file', 'node_id')

27
test/unit/container/test_server.py
View File

@ -551,33 +551,6 @@ class TestContainerController(unittest.TestCase):
{"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01"}])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self):
inbuf = StringIO()
errbuf = StringIO()

36
test/unit/container/test_updater.py
View File

@ -61,10 +61,14 @@ class TestContainerUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1)
def test_creation(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir)
@ -74,10 +78,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assert_(cu.get_account_ring() is not None)
def test_update_once_single_threaded(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded()
containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir)
@ -152,10 +160,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assertEquals(info['reported_bytes_used'], 3)
def test_unicode(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir)
subdir = os.path.join(containers_dir, 'subdir')

27
test/unit/obj/test_server.py
View File

@ -768,33 +768,6 @@ class TestObjectController(unittest.TestCase):
timestamp + '.ts')
self.assert_(os.path.isfile(objfile))
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.object_controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_call(self):
""" Test swift.object_server.ObjectController.__call__ """
inbuf = StringIO()

24
test/unit/obj/test_updater.py
View File

@ -53,10 +53,14 @@ class TestObjectUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1)
def test_creation(self):
cu = object_updater.ObjectUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'})
cu = object_updater.ObjectUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir)
@ -66,10 +70,14 @@ class TestObjectUpdater(unittest.TestCase):
self.assert_(cu.get_container_ring() is not None)
def test_update_once_single_threaded(self):
cu = object_updater.ObjectUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = object_updater.ObjectUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded()
async_dir = os.path.join(self.sda1, object_server.ASYNCDIR)
os.mkdir(async_dir)

20
test/unit/proxy/test_server.py
View File

@ -1100,17 +1100,6 @@ class TestObjectController(unittest.TestCase):
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
try:
# healthcheck test
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(body, 'OK')
# Check bad version
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
@ -1129,15 +1118,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 404'
self.assertEquals(headers[:len(exp)], exp)
# Check bad method
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 405'
self.assertEquals(headers[:len(exp)], exp)
# Check blacklist
prosrv.rate_limit_blacklist = ['a']
sock = connect_tcp(('localhost', prolis.getsockname()[1]))