openstack/swift
Code Issues Proposed changes

Initial commit of middleware refactor

Browse Source
This commit is contained in:
Chuck Thier 2010-08-20 00:42:38 +00:00
parent dab46ea017
commit 2c596c0a0f
52 changed files with 535 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bin/swift-account-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-auditor'): if c.has_section('account-auditor'):
auditor_conf = dict(c.items('account-auditor')) conf = dict(c.items('account-auditor'))
else: else:
print "Unable to find account-auditor config section in %s." % \ print "Unable to find account-auditor config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
logger = utils.get_logger(auditor_conf, 'account-auditor') logger = utils.get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda *exc_info: \ sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger) sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
auditor = AccountAuditor(server_conf, auditor_conf) auditor = AccountAuditor(conf)
if once: if once:
auditor.audit_once() auditor.audit_once()
else: else:

9
bin/swift-account-reaper
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-reaper'): if c.has_section('account-reaper'):
reaper_conf = dict(c.items('account-reaper')) conf = dict(c.items('account-reaper'))
else: else:
print "Unable to find account-reaper config section in %s." % \ print "Unable to find account-reaper config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
logger = utils.get_logger(reaper_conf, 'account-reaper') logger = utils.get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda *exc_info: \ sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger) sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
reaper = AccountReaper(server_conf, reaper_conf) reaper = AccountReaper(conf)
if once: if once:
reaper.reap_once() reaper.reap_once()
else: else:

9
bin/swift-account-replicator
View File

@ -41,17 +41,16 @@ if __name__ == '__main__':
sys.exit(1) sys.exit(1)
once = len(args) > 1 and args[1] == 'once' once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('account-server'))
if c.has_section('account-replicator'): if c.has_section('account-replicator'):
replicator_conf = dict(c.items('account-replicator')) conf = dict(c.items('account-replicator'))
else: else:
print "Unable to find account-replicator config section in %s." % \ print "Unable to find account-replicator config section in %s." % \
args[0] args[0]
sys.exit(1) sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]: if once or '--once' in [opt[0] for opt in optlist]:
AccountReplicator(server_conf, replicator_conf).replicate_once() AccountReplicator(conf).replicate_once()
else: else:
AccountReplicator(server_conf, replicator_conf).replicate_forever() AccountReplicator(conf).replicate_forever()

11
bin/swift-account-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from ConfigParser import ConfigParser
import sys import sys
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
from swift.account.server import AccountController
if __name__ == '__main__': if __name__ == '__main__':
c = ConfigParser() if len(sys.argv) != 2:
if not c.read(sys.argv[1]): print "Usage: %s CONFIG_FILE"
print "Unable to read config file." run_wsgi(sys.argv[1], default_port=6002)
sys.exit(1)
conf = dict(c.items('account-server'))
run_wsgi(AccountController, conf, default_port=6002)

11
bin/swift-auth-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from ConfigParser import ConfigParser
import sys import sys
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
from swift.auth.server import AuthController
if __name__ == '__main__': if __name__ == '__main__':
c = ConfigParser() if len(sys.argv) != 2:
if not c.read(sys.argv[1]): print "Usage: %s CONFIG_FILE"
print "Unable to read config file." run_wsgi(sys.argv[1], default_port=11000)
sys.exit(1)
conf = dict(c.items('auth-server'))
run_wsgi(AuthController, conf, default_port=11000)

9
bin/swift-container-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-auditor'): if c.has_section('container-auditor'):
auditor_conf = dict(c.items('container-auditor')) conf = dict(c.items('container-auditor'))
else: else:
print "Unable to find container-auditor config section in %s." % \ print "Unable to find container-auditor config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
logger = utils.get_logger(auditor_conf, 'container-auditor') logger = utils.get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda *exc_info: \ sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger) sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(onf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
auditor = ContainerAuditor(server_conf, auditor_conf) auditor = ContainerAuditor(conf)
if once: if once:
auditor.audit_once() auditor.audit_once()
else: else:

9
bin/swift-container-replicator
View File

@ -41,17 +41,16 @@ if __name__ == '__main__':
sys.exit(1) sys.exit(1)
once = len(args) > 1 and args[1] == 'once' once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('container-server'))
if c.has_section('container-replicator'): if c.has_section('container-replicator'):
replicator_conf = dict(c.items('container-replicator')) conf = dict(c.items('container-replicator'))
else: else:
print "Unable to find container-replicator config section in %s." % \ print "Unable to find container-replicator config section in %s." % \
args[0] args[0]
sys.exit(1) sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]: if once or '--once' in [opt[0] for opt in optlist]:
ContainerReplicator(server_conf, replicator_conf).replicate_once() ContainerReplicator(conf).replicate_once()
else: else:
ContainerReplicator(server_conf, replicator_conf).replicate_forever() ContainerReplicator(conf).replicate_forever()

11
bin/swift-container-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from ConfigParser import ConfigParser
import sys import sys
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
from swift.container.server import ContainerController
if __name__ == '__main__': if __name__ == '__main__':
c = ConfigParser() if len(sys.argv) != 2:
if not c.read(sys.argv[1]): print "Usage: %s CONFIG_FILE"
print "Unable to read config file." run_wsgi(sys.argv[1], default_port=6001)
sys.exit(1)
conf = dict(c.items('container-server'))
run_wsgi(ContainerController, conf, default_port=6001)

7
bin/swift-container-updater
View File

@ -34,15 +34,14 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-updater'): if c.has_section('container-updater'):
updater_conf = dict(c.items('container-updater')) conf = dict(c.items('container-updater'))
else: else:
print "Unable to find container-updater config section in %s." % \ print "Unable to find container-updater config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -56,7 +55,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
updater = ContainerUpdater(server_conf, updater_conf) updater = ContainerUpdater(conf)
if once: if once:
updater.update_once_single_threaded() updater.update_once_single_threaded()
else: else:

9
bin/swift-object-auditor
View File

@ -34,21 +34,20 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-auditor'): if c.has_section('object-auditor'):
auditor_conf = dict(c.items('object-auditor')) conf = dict(c.items('object-auditor'))
else: else:
print "Unable to find object-auditor config section in %s." % \ print "Unable to find object-auditor config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
logger = utils.get_logger(auditor_conf, 'object-auditor') logger = utils.get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda *exc_info: \ sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger) sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -62,7 +61,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
auditor = ObjectAuditor(server_conf, auditor_conf) auditor = ObjectAuditor(conf)
if once: if once:
auditor.audit_once() auditor.audit_once()
else: else:

39
bin/swift-object-replicator
View File

@ -27,38 +27,23 @@ from swift.common.utils import get_logger, drop_privileges, LoggerFileObject
TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes'))
def read_configs(conf_file):
c = ConfigParser()
if not c.read(conf_file):
print "Unable to read config file: %s" % conf_file
sys.exit(1)
conf = dict(c.items('object-server'))
repl_conf = dict(c.items('object-replicator'))
if not repl_conf:
sys.exit()
conf['replication_concurrency'] = repl_conf.get('concurrency',1)
conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no')
conf['daemonize'] = repl_conf.get('daemonize', 'yes')
conf['run_pause'] = repl_conf.get('run_pause', '30')
conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1')
conf['log_level'] = repl_conf.get('log_level', 'INFO')
conf['timeout'] = repl_conf.get('timeout', '5')
conf['stats_interval'] = repl_conf.get('stats_interval', '3600')
conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400))
return conf
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: if len(sys.argv) < 2:
print "Usage: object-replicator CONFIG_FILE [once]" print "Usage: object-replicator CONFIG_FILE [once]"
sys.exit() sys.exit()
try: c = ConfigParser()
conf = read_configs(sys.argv[1]) if not c.read(conf_file):
except: print "Unable to read config file: %s" % conf_file
print "Problem reading the config. Aborting object replication." sys.exit(1)
sys.exit() if c.has_section('object-replicator'):
conf = dict(c.items('object-replicator'))
else:
print "Unable to find object-replicator config section in %s" % \
sys.argv[1]
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once' once = len(sys.argv) > 2 and sys.argv[2] == 'once'
logger = get_logger(conf, 'object-replicator') logger = get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda *exc_info: \ sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)

12
bin/swift-object-server
View File

@ -14,17 +14,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from ConfigParser import ConfigParser
import sys import sys
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
from swift.obj.server import ObjectController
if __name__ == '__main__': if __name__ == '__main__':
c = ConfigParser() if len(sys.argv) != 2:
if not c.read(sys.argv[1]): print "Usage: %s CONFIG_FILE"
print "Unable to read config file." run_wsgi(sys.argv[1], default_port=6000)
sys.exit(1)
conf = dict(c.items('object-server'))
run_wsgi(ObjectController, conf, default_port=6000)

7
bin/swift-object-updater
View File

@ -34,15 +34,14 @@ if __name__ == '__main__':
print "Unable to read config file." print "Unable to read config file."
sys.exit(1) sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-updater'): if c.has_section('object-updater'):
updater_conf = dict(c.items('object-updater')) conf = dict(c.items('object-updater'))
else: else:
print "Unable to find object-updater config section in %s." % \ print "Unable to find object-updater config section in %s." % \
sys.argv[1] sys.argv[1]
sys.exit(1) sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift')) utils.drop_privileges(conf.get('user', 'swift'))
try: try:
os.setsid() os.setsid()
@ -56,7 +55,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGTERM, kill_children)
updater = ObjectUpdater(server_conf, updater_conf) updater = ObjectUpdater(conf)
if once: if once:
updater.update_once_single_threaded() updater.update_once_single_threaded()
else: else:

32
bin/swift-proxy-server
View File

@ -14,37 +14,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from ConfigParser import ConfigParser
import os
import sys import sys
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
from swift.common.auth import DevAuthMiddleware
from swift.common.memcached import MemcacheRing
from swift.common.utils import get_logger
from swift.proxy.server import Application
if __name__ == '__main__': if __name__ == '__main__':
c = ConfigParser() if len(sys.argv) != 2:
if not c.read(sys.argv[1]): print "Usage: %s CONFIG_FILE"
print "Unable to read config file." run_wsgi(sys.argv[1], default_port=80)
sys.exit(1)
conf = dict(c.items('proxy-server'))
if c.has_section('auth-server'):
auth_conf = dict(c.items('auth-server'))
else:
auth_conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
m, c = auth_conf.get('class',
'swift.common.auth.DevAuthMiddleware').rsplit('.', 1)
m = __import__(m, fromlist=[c])
authware = m.__dict__[c]
memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
logger = get_logger(conf, 'proxy')
app = Application(conf, memcache, logger)
# Wrap the app with auth
app = authware(app, auth_conf, memcache, logger)
run_wsgi(app, conf, logger=logger, default_port=80)

23
etc/account-server.conf-sample
View File

@ -1,15 +1,23 @@
[account-server] [DEFAULT]
# swift_dir = /etc/swift log_name = account
# devices = /srv/node # log_facility = LOG_LOCAL0
# mount_check = true # log_level = INFO
# bind_ip = 0.0.0.0 # bind_ip = 0.0.0.0
# bind_port = 6002 # bind_port = 6002
# workers = 1 # workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift # user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator] [account-replicator]
log_name = account-replicator
# log_facility = LOG_LOCAL0 # log_facility = LOG_LOCAL0
# log_level = INFO # log_level = INFO
# per_diff = 1000 # per_diff = 1000
@ -26,6 +34,7 @@
# reclaim_age = 86400 # reclaim_age = 86400
[account-stats] [account-stats]
log_name = account-stats
# cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 # cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31
# container_name = account_stats # container_name = account_stats
# proxy_server_conf = /etc/swift/proxy-server.conf # proxy_server_conf = /etc/swift/proxy-server.conf
@ -33,6 +42,7 @@
# log_level = INFO # log_level = INFO
[account-auditor] [account-auditor]
log_name = account-auditor
# Will audit, at most, 1 account per device per interval # Will audit, at most, 1 account per device per interval
# interval = 1800 # interval = 1800
# Maximum containers randomly picked for a given account audit # Maximum containers randomly picked for a given account audit
@ -43,6 +53,7 @@
# log_level = INFO # log_level = INFO
[account-reaper] [account-reaper]
log_name = account-reaper
# concurrency = 25 # concurrency = 25
# interval = 3600 # interval = 3600
# node_timeout = 10 # node_timeout = 10

17
etc/auth-server.conf-sample
View File

@ -1,15 +1,22 @@
[auth-server] [DEFAULT]
# swift_dir = /etc/swift log_name = auth
# bind_ip = 0.0.0.0 # bind_ip = 0.0.0.0
# bind_port = 11000 # bind_port = 11000
# log_facility = LOG_LOCAL0 # log_facility = LOG_LOCAL0
# log_level = INFO # log_level = INFO
# workers = 1 # workers = 1
# user = swift
# swift_dir = /etc/swift
# cert_file = Default is no cert; format is path like /etc/swift/auth.crt
# key_file = Default is no key; format is path like /etc/swift/auth.key
[pipeline:main]
pipeline = auth-server
[app:auth-server]
use = egg:swift#auth
# reseller_prefix = AUTH # reseller_prefix = AUTH
# default_cluster_url = http://127.0.0.1:9000/v1 # default_cluster_url = http://127.0.0.1:9000/v1
# token_life = 86400 # token_life = 86400
# log_headers = False # log_headers = False
# cert_file = Default is no cert; format is path like /etc/swift/auth.crt
# key_file = Default is no key; format is path like /etc/swift/auth.key
# node_timeout = 10 # node_timeout = 10
user = swift

28
etc/container-server.conf-sample
View File

@ -1,19 +1,25 @@
[container-server] [DEFAULT]
# swift_dir = /etc/swift log_name = container
# devices = /srv/node # log_facility = LOG_LOCAL0
# mount_check = true # log_level = INFO
# bind_ip = 0.0.0.0 # bind_ip = 0.0.0.0
# bind_port = 6001 # bind_port = 6001
# workers = 1 # workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift # user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
# node_timeout = 3 # node_timeout = 3
# conn_timeout = 0.5 # conn_timeout = 0.5
[container-replicator] [container-replicator]
# log_facility = LOG_LOCAL0 log_name = container-replicator
# log_level = INFO
# per_diff = 1000 # per_diff = 1000
# concurrency = 8 # concurrency = 8
# run_pause = 30 # run_pause = 30
@ -23,21 +29,19 @@
# reclaim_age = 604800 # reclaim_age = 604800
[container-updater] [container-updater]
log_name = container-updater
# interval = 300 # interval = 300
# concurrency = 4 # concurrency = 4
# node_timeout = 3 # node_timeout = 3
# conn_timeout = 0.5 # conn_timeout = 0.5
# slowdown will sleep that amount between containers # slowdown will sleep that amount between containers
# slowdown = 0.01 # slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[container-auditor] [container-auditor]
log_name = container-auditor
# Will audit, at most, 1 container per device per interval # Will audit, at most, 1 container per device per interval
# interval = 1800 # interval = 1800
# Maximum objects randomly picked for a given container audit # Maximum objects randomly picked for a given container audit
# max_object_count = 100 # max_object_count = 100
# node_timeout = 10 # node_timeout = 10
# conn_timeout = 0.5 # conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

34
etc/object-server.conf-sample
View File

@ -1,24 +1,30 @@
[object-server] [DEFAULT]
# swift_dir = /etc/swift log_name = object
# devices = /srv/node # log_facility = LOG_LOCAL0
# mount_check = true # log_level = INFO
# bind_ip = 0.0.0.0 # bind_ip = 0.0.0.0
# bind_port = 6000 # bind_port = 6000
# workers = 1 # workers = 1
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_requests = True
# user = swift # user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
# log_requests = True
# node_timeout = 3 # node_timeout = 3
# conn_timeout = 0.5 # conn_timeout = 0.5
# network_chunk_size = 8192 # network_chunk_size = 65536
# disk_chunk_size = 32768 # disk_chunk_size = 65536
# max_upload_time = 86400 # max_upload_time = 86400
# slow = 1 # slow = 1
[object-replicator] [object-replicator]
# log_facility = LOG_LOCAL0 log_name = object-replicator
# log_level = INFO
# daemonize = on # daemonize = on
# run_pause = 30 # run_pause = 30
# concurrency = 1 # concurrency = 1
@ -28,19 +34,17 @@
# reclaim_age = 604800 # reclaim_age = 604800
[object-updater] [object-updater]
log_name = object-updater
# interval = 300 # interval = 300
# concurrency = 1 # concurrency = 1
# node_timeout = 10 # node_timeout = 10
# conn_timeout = 0.5 # conn_timeout = 0.5
# slowdown will sleep that amount between objects # slowdown will sleep that amount between objects
# slowdown = 0.01 # slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[object-auditor] [object-auditor]
log_name = object-auditor
# Will audit, at most, 1 object per device per interval # Will audit, at most, 1 object per device per interval
# interval = 1800 # interval = 1800
# node_timeout = 10 # node_timeout = 10
# conn_timeout = 0.5 # conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

38
etc/proxy-server.conf-sample
View File

@ -1,21 +1,26 @@
[proxy-server] [DEFAULT]
# bind_ip = 0.0.0.0 log_name = proxy
# bind_port = 80
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
# swift_dir = /etc/swift
# log_facility = LOG_LOCAL0 # log_facility = LOG_LOCAL0
# log_level = INFO # log_level = INFO
# log_headers = False # bind_ip = 0.0.0.0
# bind_port = 80
# workers = 1 # workers = 1
# user = swift # user = swift
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
[pipeline:main]
pipeline = healthcheck cache auth proxy
[app:proxy]
use = egg:swift#proxy
filter-with = auth
# swift_dir = /etc/swift
# log_headers = False
# recheck_account_existence = 60 # recheck_account_existence = 60
# recheck_container_existence = 60 # recheck_container_existence = 60
# object_chunk_size = 8192 # object_chunk_size = 8192
# client_chunk_size = 8192 # client_chunk_size = 8192
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211
# node_timeout = 10 # node_timeout = 10
# client_timeout = 60 # client_timeout = 60
# conn_timeout = 0.5 # conn_timeout = 0.5
@ -31,8 +36,17 @@
# rate_limit_account_whitelist = acct1,acct2,etc # rate_limit_account_whitelist = acct1,acct2,etc
# rate_limit_account_blacklist = acct3,acct4,etc # rate_limit_account_blacklist = acct3,acct4,etc
# [auth-server] [filter:auth]
# class = swift.common.auth.DevAuthMiddleware use = egg:swift#auth
# ip = 127.0.0.1 # ip = 127.0.0.1
# port = 11000 # port = 11000
# node_timeout = 10 # node_timeout = 10
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#cache
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211

51
setup.py
View File

@ -55,21 +55,38 @@ setup(
'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.6',
'Environment :: No Input/Output (Daemon)', 'Environment :: No Input/Output (Daemon)',
], ],
install_requires=[], # removed for better compat install_requires=[], # removed for better compat
scripts=['bin/st', 'bin/swift-account-auditor', scripts=[
'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/st', 'bin/swift-account-auditor',
'bin/swift-account-replicator', 'bin/swift-account-server', 'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-auth-create-account', 'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', 'bin/swift-auth-create-account',
'bin/swift-container-auditor', 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-replicator', 'bin/swift-container-auditor',
'bin/swift-container-server', 'bin/swift-container-updater', 'bin/swift-container-replicator',
'bin/swift-drive-audit', 'bin/swift-get-nodes', 'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-init', 'bin/swift-object-auditor', 'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-object-info', 'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-replicator', 'bin/swift-object-info',
'bin/swift-object-server', 'bin/swift-object-replicator',
'bin/swift-object-updater', 'bin/swift-proxy-server', 'bin/swift-object-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate', 'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-stats-report']) 'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-stats-report'
],
entry_points={
'paste.app_factory' : [
'proxy=swift.proxy.server:app_factory',
'object=swift.obj.server:app_factory',
'container=swift.container.server:app_factory',
'account=swift.account.server:app_factory',
'auth=swift.auth.server:app_factory',
],
'paste.filter_factory' : [
'auth=swift.common.middleware.auth:filter_factory',
'healthcheck=swift.common.middleware.healthcheck:filter_factory',
'cache=swift.common.middleware.cache:filter_factory',
],
},
)

18
swift/account/auditor.py
View File

@ -35,19 +35,19 @@ class AuditException(Exception):
class AccountAuditor(object): class AccountAuditor(object):
"""Audit accounts.""" """Audit accounts."""
def __init__(self, server_conf, auditor_conf): def __init__(self, conf):
self.logger = get_logger(auditor_conf, 'account-auditor') self.logger = get_logger(conf, 'account-auditor')
self.devices = server_conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800)) self.interval = int(conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_container_count = \ self.max_container_count = \
int(auditor_conf.get('max_container_count', 100)) int(conf.get('max_container_count', 100))
self.container_passes = 0 self.container_passes = 0
self.container_failures = 0 self.container_failures = 0
self.container_errors = 0 self.container_errors = 0

20
swift/account/reaper.py
View File

@ -50,25 +50,23 @@ class AccountReaper(object):
configuration parameters. configuration parameters.
""" """
log_name = 'account-reaper' def __init__(self, conf):
self.logger = get_logger(conf)
def __init__(self, server_conf, reaper_conf): self.devices = conf.get('devices', '/srv/node')
self.logger = get_logger(reaper_conf, self.log_name) self.mount_check = conf.get('mount_check', 'true').lower() in \
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(reaper_conf.get('interval', 3600)) self.interval = int(conf.get('interval', 3600))
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.account_ring = None self.account_ring = None
self.container_ring = None self.container_ring = None
self.object_ring = None self.object_ring = None
self.node_timeout = int(reaper_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.myips = whataremyips() self.myips = whataremyips()
self.concurrency = int(reaper_conf.get('concurrency', 25)) self.concurrency = int(conf.get('concurrency', 25))
self.container_concurrency = self.object_concurrency = \ self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency) sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency) self.container_pool = GreenPool(size=self.container_concurrency)

16
swift/account/server.py
View File

@ -31,10 +31,9 @@ from xml.sax import saxutils
from swift.common.db import AccountBroker from swift.common.db import AccountBroker
from swift.common.utils import get_param, split_path, storage_directory, \ from swift.common.utils import get_param, split_path, storage_directory, \
hash_path hash_path, get_logger
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable check_mount, check_float, check_xml_encodable
from swift.common.healthcheck import healthcheck
from swift.common.db_replicator import ReplicatorRpc from swift.common.db_replicator import ReplicatorRpc
@ -43,10 +42,9 @@ DATADIR = 'accounts'
class AccountController(object): class AccountController(object):
"""WSGI controller for the account server.""" """WSGI controller for the account server."""
log_name = 'account'
def __init__(self, conf): def __init__(self, conf):
self.logger = get_logger(conf, self.log_name) self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node') self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
@ -253,9 +251,7 @@ class AccountController(object):
def __call__(self, env, start_response): def __call__(self, env, start_response):
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
if req.path_info == '/healthcheck': if not check_xml_encodable(req.path_info):
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
try: try:
@ -288,3 +284,9 @@ class AccountController(object):
else: else:
self.logger.info(log_message) self.logger.info(log_message)
return res(env, start_response) return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI account server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return AccountController(conf)

9
swift/auth/server.py
View File

@ -89,10 +89,8 @@ class AuthController(object):
configuration parameters. configuration parameters.
""" """
log_name = 'auth'
def __init__(self, conf, ring=None): def __init__(self, conf, ring=None):
self.logger = get_logger(conf, self.log_name) self.logger = get_logger(conf)
self.swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.default_cluster_url = \ self.default_cluster_url = \
conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1') conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1')
@ -500,3 +498,8 @@ class AuthController(object):
def __call__(self, env, start_response): def __call__(self, env, start_response):
""" Used by the eventlet.wsgi.server """ """ Used by the eventlet.wsgi.server """
return self.handleREST(env, start_response) return self.handleREST(env, start_response)
def app_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
return AuthController(conf)

26
swift/common/db_replicator.py
View File

@ -89,28 +89,28 @@ class Replicator(object):
Implements the logic for directing db replication. Implements the logic for directing db replication.
""" """
def __init__(self, server_conf, replicator_conf): def __init__(self, conf):
self.logger = \ self.logger = \
get_logger(replicator_conf, '%s-replicator' % self.server_type) get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda * exc_info: \ sys.excepthook = lambda * exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(self.logger) sys.stdout = sys.stderr = LoggerFileObject(self.logger)
self.root = server_conf.get('devices', '/srv/node') self.root = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
self.port = int(server_conf.get('bind_port', self.default_port)) self.port = int(conf.get('bind_port', self.default_port))
concurrency = int(replicator_conf.get('concurrency', 8)) concurrency = int(conf.get('concurrency', 8))
self.cpool = GreenPool(size=concurrency) self.cpool = GreenPool(size=concurrency)
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file)) self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file))
self.per_diff = int(replicator_conf.get('per_diff', 1000)) self.per_diff = int(conf.get('per_diff', 1000))
self.run_pause = int(replicator_conf.get('run_pause', 30)) self.run_pause = int(conf.get('run_pause', 30))
self.vm_test_mode = replicator_conf.get( self.vm_test_mode = conf.get(
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.node_timeout = int(replicator_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7)) self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
self._zero_stats() self._zero_stats()
def _zero_stats(self): def _zero_stats(self):

29
swift/common/healthcheck.py
View File

@ -1,29 +0,0 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Response
class HealthCheckController(object):
"""Basic controller used for monitoring."""
def __init__(self, *args, **kwargs):
pass
@classmethod
def GET(self, req):
return Response(request=req, body="OK", content_type="text/plain")
healthcheck = HealthCheckController.GET

0
swift/common/middleware/__init__.py Normal file
View File

45
swift/common/auth.py → swift/common/middleware/auth.py
View File

@ -21,6 +21,8 @@ from eventlet.timeout import Timeout
from swift.common.utils import split_path from swift.common.utils import split_path
from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import get_logger, cache_from_env
from swift.common.memcached import MemcacheRing
class DevAuthMiddleware(object): class DevAuthMiddleware(object):
@ -28,33 +30,37 @@ class DevAuthMiddleware(object):
Auth Middleware that uses the dev auth server Auth Middleware that uses the dev auth server
""" """
def __init__(self, app, conf, memcache_client, logger): def __init__(self, app, conf, memcache_client=None, logger=None):
self.app = app self.app = app
self.memcache_client = memcache_client self.memcache_client = memcache_client
self.logger = logger if logger is None:
self.logger = get_logger(conf)
else:
self.logger = logger
self.conf = conf self.conf = conf
self.auth_host = conf.get('ip', '127.0.0.1') self.auth_host = conf.get('ip', '127.0.0.1')
self.auth_port = int(conf.get('port', 11000)) self.auth_port = int(conf.get('port', 11000))
self.timeout = int(conf.get('node_timeout', 10)) self.timeout = int(conf.get('node_timeout', 10))
def __call__(self, env, start_response): def __call__(self, env, start_response):
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
req = Request(env) req = Request(env)
if req.path != '/healthcheck': if 'x-storage-token' in req.headers and \
if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers:
'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token']
req.headers['x-auth-token'] = req.headers['x-storage-token'] try:
version, account, container, obj = split_path(req.path, 1, 4, True) version, account, container, obj = split_path(req.path, 1, 4, True)
if account is None: except ValueError, e:
return HTTPPreconditionFailed(request=req, body='Bad URL')( version = account = container = obj = None
env, start_response) if account is None:
if not req.headers.get('x-auth-token'): return HTTPPreconditionFailed(request=req, body='Bad URL')(
return HTTPPreconditionFailed(request=req, env, start_response)
body='Missing Auth Token')(env, start_response) if not req.headers.get('x-auth-token'):
if account is None: return HTTPPreconditionFailed(request=req,
return HTTPPreconditionFailed( body='Missing Auth Token')(env, start_response)
request=req, body='Bad URL')(env, start_response) if not self.auth(account, req.headers['x-auth-token']):
if not self.auth(account, req.headers['x-auth-token']): return HTTPUnauthorized(request=req)(env, start_response)
return HTTPUnauthorized(request=req)(env, start_response)
# If we get here, then things should be good. # If we get here, then things should be good.
return self.app(env, start_response) return self.app(env, start_response)
@ -95,3 +101,8 @@ class DevAuthMiddleware(object):
val = (now, validated) val = (now, validated)
self.memcache_client.set(key, val, timeout=validated) self.memcache_client.set(key, val, timeout=validated)
return True return True
def filter_factory(global_conf, **local_conf):
def auth_filter(app):
return DevAuthMiddleware(app, local_conf)
return auth_filter

34
swift/common/middleware/cache.py Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.common.memcached import MemcacheRing
class CacheMiddleware(object):
"""
Caching middleware that manages caching in swift.
"""
def __init__(self, app, conf):
self.app = app
self.memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
def __call__(self, env, start_response):
env['swift.cache'] = self.memcache
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def cache_filter(app):
return CacheMiddleware(app, local_conf)
return cache_filter

43
swift/common/middleware/healthcheck.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Request, Response
class HealthCheckMiddleware(object):
"""
Healthcheck middleware used for monitoring.
If the path is /healthcheck, it will respond with "OK" in the body
"""
def __init__(self, app, *args, **kwargs):
self.app = app
def GET(self, req):
"""Returns a 200 response with "OK" in the body."""
return Response(request=req, body="OK", content_type="text/plain")
def __call__(self, env, start_response):
req = Request(env)
if req.path == '/healthcheck':
return self.GET(req)(env, start_response)
else:
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def healthcheck_filter(app):
return HealthCheckMiddleware(app)
return healthcheck_filter

13
swift/common/utils.py
View File

@ -323,7 +323,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args) call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name): def get_logger(conf, name=None):
""" """
Get the current system logger using config settings. Get the current system logger using config settings.
@ -342,6 +342,8 @@ def get_logger(conf, name):
if conf is None: if conf is None:
root_logger.setLevel(logging.INFO) root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name) return NamedLogger(root_logger, name)
if name is None:
name = conf.get('log_name', 'swift')
get_logger.handler = SysLogHandler(address='/dev/log', get_logger.handler = SysLogHandler(address='/dev/log',
facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'),
SysLogHandler.LOG_LOCAL0)) SysLogHandler.LOG_LOCAL0))
@ -513,3 +515,12 @@ def unlink_older_than(path, mtime):
os.unlink(fpath) os.unlink(fpath)
except OSError: except OSError:
pass pass
def item_from_env(env, item_name):
item = env.get(item_name, None)
if item is None:
logging.error("ERROR: %s could not be found in env!" % item_name)
return item
def cache_from_env(env):
return item_from_env(env, 'swift.cache')

18
swift/common/wsgi.py
View File

@ -24,6 +24,7 @@ import mimetools
import eventlet import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen from eventlet import greenio, GreenPool, sleep, wsgi, listen
from paste.deploy import loadapp, appconfig
# Hook to ensure connection resets don't blow up our servers. # Hook to ensure connection resets don't blow up our servers.
# Remove with next release of Eventlet that has it in the set already. # Remove with next release of Eventlet that has it in the set already.
@ -58,18 +59,24 @@ def monkey_patch_mimetools():
# We might be able to pull pieces of this out to test, but right now it seems # We might be able to pull pieces of this out to test, but right now it seems
# like more work than it's worth. # like more work than it's worth.
def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover def run_wsgi(conf_file, *args, **kwargs): # pragma: no cover
""" """
Loads common settings from conf, then instantiates app and runs Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers. the server using the specified number of workers.
:param app: WSGI callable :param conf_file: Path to paste.deploy style configuration file
:param conf: Configuration dictionary
""" """
try:
app = loadapp('config:%s' % conf_file)
conf = appconfig('config:%s' % conf_file)
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs: if 'logger' in kwargs:
logger = kwargs['logger'] logger = kwargs['logger']
else: else:
logger = get_logger(conf, app.log_name) logger = get_logger(conf)
# log uncaught exceptions # log uncaught exceptions
sys.excepthook = lambda * exc_info: \ sys.excepthook = lambda * exc_info: \
@ -103,9 +110,6 @@ def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1')) worker_count = int(conf.get('workers', '1'))
drop_privileges(conf.get('user', 'swift')) drop_privileges(conf.get('user', 'swift'))
if isinstance(app, type):
# Instantiate app if it hasn't been already
app = app(conf, *args)
def run_server(): def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0" wsgi.HttpProtocol.default_request_version = "HTTP/1.0"

18
swift/container/auditor.py
View File

@ -36,20 +36,20 @@ class AuditException(Exception):
class ContainerAuditor(object): class ContainerAuditor(object):
"""Audit containers.""" """Audit containers."""
def __init__(self, server_conf, auditor_conf): def __init__(self, conf):
self.logger = get_logger(auditor_conf, 'container-auditor') self.logger = get_logger(conf)
self.devices = server_conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800)) self.interval = int(conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None self.account_ring = None
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.object_ring = None self.object_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_object_count = int(auditor_conf.get('max_object_count', 100)) self.max_object_count = int(conf.get('max_object_count', 100))
self.account_passes = 0 self.account_passes = 0
self.account_failures = 0 self.account_failures = 0
self.account_errors = 0 self.account_errors = 0

17
swift/container/server.py
View File

@ -31,11 +31,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
from swift.common.db import ContainerBroker from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.utils import get_logger, get_param, hash_path, \
storage_directory, split_path storage_directory, split_path, get_logger
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable check_mount, check_float, check_xml_encodable
from swift.common.bufferedhttp import http_connect from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import healthcheck
from swift.common.exceptions import ConnectionTimeout from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc from swift.common.db_replicator import ReplicatorRpc
@ -45,10 +44,8 @@ DATADIR = 'containers'
class ContainerController(object): class ContainerController(object):
"""WSGI Controller for the container server.""" """WSGI Controller for the container server."""
log_name = 'container'
def __init__(self, conf): def __init__(self, conf):
self.logger = get_logger(conf, self.log_name) self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node/') self.root = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
@ -347,9 +344,7 @@ class ContainerController(object):
def __call__(self, env, start_response): def __call__(self, env, start_response):
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
if req.path_info == '/healthcheck': if not check_xml_encodable(req.path_info):
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
try: try:
@ -378,3 +373,9 @@ class ContainerController(object):
else: else:
self.logger.info(log_message) self.logger.info(log_message)
return res(env, start_response) return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI container server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ContainerController(conf)

20
swift/container/updater.py
View File

@ -33,19 +33,19 @@ from swift.common.utils import get_logger, whataremyips
class ContainerUpdater(object): class ContainerUpdater(object):
"""Update container information in account listings.""" """Update container information in account listings."""
def __init__(self, server_conf, updater_conf): def __init__(self, conf):
self.logger = get_logger(updater_conf, 'container-updater') self.logger = get_logger(conf, 'container-updater')
self.devices = server_conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300)) self.interval = int(conf.get('interval', 300))
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None self.account_ring = None
self.concurrency = int(updater_conf.get('concurrency', 4)) self.concurrency = int(conf.get('concurrency', 4))
self.slowdown = float(updater_conf.get('slowdown', 0.01)) self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 3)) self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.no_changes = 0 self.no_changes = 0
self.successes = 0 self.successes = 0
self.failures = 0 self.failures = 0

16
swift/obj/auditor.py
View File

@ -33,17 +33,17 @@ from swift.common.exceptions import AuditException
class ObjectAuditor(object): class ObjectAuditor(object):
"""Audit objects.""" """Audit objects."""
def __init__(self, server_conf, auditor_conf): def __init__(self, conf):
self.logger = get_logger(auditor_conf, 'object-auditor') self.logger = get_logger(conf)
self.devices = server_conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800)) self.interval = int(conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.passes = 0 self.passes = 0
self.quarantines = 0 self.quarantines = 0
self.errors = 0 self.errors = 0

2
swift/obj/replicator.py
View File

@ -215,7 +215,7 @@ class ObjectReplicator(object):
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.port = int(conf.get('bind_port', 6000)) self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('replication_concurrency', 1)) self.concurrency = int(conf.get('concurrency', 1))
self.timeout = conf['timeout'] self.timeout = conf['timeout']
self.stats_interval = int(conf['stats_interval']) self.stats_interval = int(conf['stats_interval'])
self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz'))

19
swift/obj/server.py
View File

@ -36,9 +36,8 @@ from xattr import getxattr, setxattr
from eventlet import sleep, Timeout from eventlet import sleep, Timeout
from swift.common.utils import mkdirs, normalize_timestamp, \ from swift.common.utils import mkdirs, normalize_timestamp, \
storage_directory, hash_path, get_logger, renamer, fallocate, \ storage_directory, hash_path, renamer, fallocate, \
split_path, drop_buffer_cache split_path, drop_buffer_cache, get_logger
from swift.common.healthcheck import healthcheck
from swift.common.bufferedhttp import http_connect from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, check_mount, \ from swift.common.constraints import check_object_creation, check_mount, \
check_float, check_xml_encodable check_float, check_xml_encodable
@ -242,8 +241,6 @@ class DiskFile(object):
class ObjectController(object): class ObjectController(object):
"""Implements the WSGI application for the Swift Object Server.""" """Implements the WSGI application for the Swift Object Server."""
log_name = 'object'
def __init__(self, conf): def __init__(self, conf):
""" """
Creates a new WSGI application for the Swift Object Server. An Creates a new WSGI application for the Swift Object Server. An
@ -251,7 +248,7 @@ class ObjectController(object):
<source-dir>/etc/object-server.conf-sample or <source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample. /etc/swift/object-server.conf-sample.
""" """
self.logger = get_logger(conf, self.log_name) self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node/') self.devices = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
@ -560,9 +557,7 @@ class ObjectController(object):
"""WSGI Application entry point for the Swift Object Server.""" """WSGI Application entry point for the Swift Object Server."""
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
if req.path_info == '/healthcheck': if not check_xml_encodable(req.path_info):
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
try: try:
@ -596,3 +591,9 @@ class ObjectController(object):
if slow > 0: if slow > 0:
sleep(slow) sleep(slow)
return res(env, start_response) return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI object server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ObjectController(conf)

20
swift/obj/updater.py
View File

@ -32,19 +32,19 @@ from swift.obj.server import ASYNCDIR
class ObjectUpdater(object): class ObjectUpdater(object):
"""Update object information in container listings.""" """Update object information in container listings."""
def __init__(self, server_conf, updater_conf): def __init__(self, conf):
self.logger = get_logger(updater_conf, 'object-updater') self.logger = get_logger(conf)
self.devices = server_conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300)) self.interval = int(conf.get('interval', 300))
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None self.container_ring = None
self.concurrency = int(updater_conf.get('concurrency', 1)) self.concurrency = int(conf.get('concurrency', 1))
self.slowdown = float(updater_conf.get('slowdown', 0.01)) self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 10)) self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.successes = 0 self.successes = 0
self.failures = 0 self.failures = 0

33
swift/proxy/server.py
View File

@ -31,9 +31,9 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \
from webob import Request, Response from webob import Request, Response
from swift.common.ring import Ring from swift.common.ring import Ring
from swift.common.utils import get_logger, normalize_timestamp, split_path from swift.common.utils import get_logger, normalize_timestamp, split_path, \
cache_from_env
from swift.common.bufferedhttp import http_connect from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import HealthCheckController
from swift.common.constraints import check_object_creation, check_metadata, \ from swift.common.constraints import check_object_creation, check_metadata, \
MAX_FILE_SIZE, check_xml_encodable MAX_FILE_SIZE, check_xml_encodable
from swift.common.exceptions import ChunkReadTimeout, \ from swift.common.exceptions import ChunkReadTimeout, \
@ -47,7 +47,7 @@ def update_headers(response, headers):
Helper function to update headers in the response. Helper function to update headers in the response.
:param response: webob.Response object :param response: webob.Response object
:param headers: dictionary headers :parm headers: dictionary headers
""" """
if hasattr(headers, 'items'): if hasattr(headers, 'items'):
headers = headers.items() headers = headers.items()
@ -930,14 +930,12 @@ class AccountController(Controller):
class BaseApplication(object): class BaseApplication(object):
"""Base WSGI application for the proxy server""" """Base WSGI application for the proxy server"""
log_name = 'base_application' def __init__(self, conf, memcache=None, logger=None, account_ring=None,
def __init__(self, conf, memcache, logger=None, account_ring=None,
container_ring=None, object_ring=None): container_ring=None, object_ring=None):
if logger: if logger is None:
self.logger = logger self.logger = get_logger(conf)
else: else:
self.logger = get_logger(conf, self.log_name) self.logger = logger
if conf is None: if conf is None:
conf = {} conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
@ -991,8 +989,6 @@ class BaseApplication(object):
return ContainerController, d return ContainerController, d
elif account and not container and not obj: elif account and not container and not obj:
return AccountController, d return AccountController, d
elif version and version == 'healthcheck':
return HealthCheckController, d
return None, d return None, d
def __call__(self, env, start_response): def __call__(self, env, start_response):
@ -1004,6 +1000,8 @@ class BaseApplication(object):
:param start_response: WSGI callable :param start_response: WSGI callable
""" """
try: try:
if self.memcache is None:
self.memcache = cache_from_env(env)
req = self.update_request(Request(env)) req = self.update_request(Request(env))
if 'eventlet.posthooks' in env: if 'eventlet.posthooks' in env:
env['eventlet.posthooks'].append( env['eventlet.posthooks'].append(
@ -1048,13 +1046,6 @@ class BaseApplication(object):
controller, path_parts = self.get_controller(req.path) controller, path_parts = self.get_controller(req.path)
except ValueError: except ValueError:
return HTTPNotFound(request=req) return HTTPNotFound(request=req)
if controller == HealthCheckController:
controller = controller(self, **path_parts)
controller.trans_id = req.headers.get('x-cf-trans-id', '-')
if req.method == 'GET':
return controller.GET(req)
return HTTPMethodNotAllowed(request=req)
if not check_xml_encodable(req.path_info): if not check_xml_encodable(req.path_info):
return HTTPPreconditionFailed(request=req, body='Invalid UTF8') return HTTPPreconditionFailed(request=req, body='Invalid UTF8')
if not controller: if not controller:
@ -1086,8 +1077,6 @@ class BaseApplication(object):
class Application(BaseApplication): class Application(BaseApplication):
"""WSGI application for the proxy server.""" """WSGI application for the proxy server."""
log_name = 'proxy'
def handle_request(self, req): def handle_request(self, req):
""" """
Wraps the BaseApplication.handle_request and logs the request. Wraps the BaseApplication.handle_request and logs the request.
@ -1181,3 +1170,7 @@ class Application(BaseApplication):
return Response(status='498 Rate Limited', return Response(status='498 Rate Limited',
body='Slow down', request=req) body='Slow down', request=req)
return None return None
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
return Application(local_conf)

27
test/unit/account/test_server.py
View File

@ -795,33 +795,6 @@ class TestAccountController(unittest.TestCase):
listing.append(node2.firstChild.nodeValue) listing.append(node2.firstChild.nodeValue)
self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2']) self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2'])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self): def test_through_call(self):
inbuf = StringIO() inbuf = StringIO()
errbuf = StringIO() errbuf = StringIO()

2
test/unit/auth/test_server.py
View File

@ -75,7 +75,7 @@ class TestAuthServer(unittest.TestCase):
'auth_server') 'auth_server')
rmtree(self.testdir, ignore_errors=1) rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir) os.mkdir(self.testdir)
self.conf = {'swift_dir': self.testdir} self.conf = {'swift_dir': self.testdir, 'log_name': 'auth'}
self.controller = auth_server.AuthController(self.conf, FakeRing()) self.controller = auth_server.AuthController(self.conf, FakeRing())
def tearDown(self): def tearDown(self):

0
test/unit/common/middleware/__init__.py Normal file
View File

2
test/unit/common/test_auth.py → test/unit/common/middleware/test_auth.py
View File

@ -23,7 +23,7 @@ from contextlib import contextmanager
import eventlet import eventlet
from webob import Request from webob import Request
from swift.common import auth from swift.common.middleware import auth
# mocks # mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

42
test/unit/common/middleware/test_cache.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from webob import Request
from swift.common.middleware import cache
from swift.common.memcached import MemcacheRing
class FakeApp(object):
def __call__(self, env, start_response):
return env
def start_response(*args):
pass
class TestCacheMiddleware(unittest.TestCase):
def setUp(self):
self.app = cache.CacheMiddleware(FakeApp(), {})
def test_cache_middleware(self):
req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertTrue('swift.cache' in resp)
self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing))
if __name__ == '__main__':
unittest.main()

24
test/unit/common/test_healthcheck.py → test/unit/common/middleware/test_healthcheck.py
View File

@ -17,17 +17,29 @@ import unittest
from webob import Request from webob import Request
from swift.common import healthcheck from swift.common.middleware import healthcheck
class FakeApp(object):
def __call__(self, env, start_response):
return "FAKE APP"
def start_response(*args):
pass
class TestHealthCheck(unittest.TestCase): class TestHealthCheck(unittest.TestCase):
def test_healthcheck(self): def setUp(self):
controller = healthcheck.HealthCheckController() self.app = healthcheck.HealthCheckMiddleware(FakeApp())
req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'})
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
def test_healthcheck(self):
req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['OK'])
def test_healtcheck_pass(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

16
test/unit/common/test_db_replicator.py
View File

@ -139,7 +139,7 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals(conn.post(1, 2, 3), None) self.assertEquals(conn.post(1, 2, 3), None)
def test_rsync_file(self): def test_rsync_file(self):
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
with _mock_process(-1): with _mock_process(-1):
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
self.assertEquals(False, self.assertEquals(False,
@ -150,13 +150,13 @@ class TestDBReplicator(unittest.TestCase):
replicator._rsync_file('/some/file', 'remote:/some/file')) replicator._rsync_file('/some/file', 'remote:/some/file'))
def test_rsync_db(self): def test_rsync_db(self):
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
replicator._rsync_file = lambda *args: True replicator._rsync_file = lambda *args: True
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd') replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd')
def test_in_sync(self): def test_in_sync(self):
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
self.assertEquals(replicator._in_sync( self.assertEquals(replicator._in_sync(
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
@ -171,16 +171,16 @@ class TestDBReplicator(unittest.TestCase):
FakeBroker(), -1)), False) FakeBroker(), -1)), False)
def test_replicate_once(self): def test_replicate_once(self):
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
replicator.replicate_once() replicator.replicate_once()
def test_usync(self): def test_usync(self):
fake_http = PostReplHttp() fake_http = PostReplHttp()
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890') replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')
def test_repl_to_node(self): def test_repl_to_node(self):
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000} fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000}
fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b', fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b',
'created_at': 100, 'put_timestamp': 0, 'created_at': 100, 'put_timestamp': 0,
@ -192,13 +192,13 @@ class TestDBReplicator(unittest.TestCase):
def test_stats(self): def test_stats(self):
# I'm not sure how to test that this logs the right thing, # I'm not sure how to test that this logs the right thing,
# but we can at least make sure it gets covered. # but we can at least make sure it gets covered.
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
replicator._zero_stats() replicator._zero_stats()
replicator._report_stats() replicator._report_stats()
def test_replicate_object(self): def test_replicate_object(self):
db_replicator.lock_parent_directory = lock_parent_directory db_replicator.lock_parent_directory = lock_parent_directory
replicator = TestReplicator({}, {}) replicator = TestReplicator({})
replicator._replicate_object('0', 'file', 'node_id') replicator._replicate_object('0', 'file', 'node_id')

27
test/unit/container/test_server.py
View File

@ -551,33 +551,6 @@ class TestContainerController(unittest.TestCase):
{"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain", {"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01"}]) "last_modified":"1970-01-01T00:00:01"}])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self): def test_through_call(self):
inbuf = StringIO() inbuf = StringIO()
errbuf = StringIO() errbuf = StringIO()

36
test/unit/container/test_updater.py
View File

@ -61,10 +61,14 @@ class TestContainerUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1) rmtree(self.testdir, ignore_errors=1)
def test_creation(self): def test_creation(self):
cu = container_updater.ContainerUpdater( cu = container_updater.ContainerUpdater({
{'devices': self.devices_dir, 'mount_check': 'false', 'devices': self.devices_dir,
'swift_dir': self.testdir}, 'mount_check': 'false',
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) 'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger')) self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None) self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir) self.assertEquals(cu.devices, self.devices_dir)
@ -74,10 +78,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assert_(cu.get_account_ring() is not None) self.assert_(cu.get_account_ring() is not None)
def test_update_once_single_threaded(self): def test_update_once_single_threaded(self):
cu = container_updater.ContainerUpdater( cu = container_updater.ContainerUpdater({
{'devices': self.devices_dir, 'mount_check': 'false', 'devices': self.devices_dir,
'swift_dir': self.testdir}, 'mount_check': 'false',
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) 'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded() cu.update_once_single_threaded()
containers_dir = os.path.join(self.sda1, container_server.DATADIR) containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir) os.mkdir(containers_dir)
@ -152,10 +160,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assertEquals(info['reported_bytes_used'], 3) self.assertEquals(info['reported_bytes_used'], 3)
def test_unicode(self): def test_unicode(self):
cu = container_updater.ContainerUpdater( cu = container_updater.ContainerUpdater({
{'devices': self.devices_dir, 'mount_check': 'false', 'devices': self.devices_dir,
'swift_dir': self.testdir}, 'mount_check': 'false',
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) 'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
containers_dir = os.path.join(self.sda1, container_server.DATADIR) containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir) os.mkdir(containers_dir)
subdir = os.path.join(containers_dir, 'subdir') subdir = os.path.join(containers_dir, 'subdir')

27
test/unit/obj/test_server.py
View File

@ -768,33 +768,6 @@ class TestObjectController(unittest.TestCase):
timestamp + '.ts') timestamp + '.ts')
self.assert_(os.path.isfile(objfile)) self.assert_(os.path.isfile(objfile))
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.object_controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_call(self): def test_call(self):
""" Test swift.object_server.ObjectController.__call__ """ """ Test swift.object_server.ObjectController.__call__ """
inbuf = StringIO() inbuf = StringIO()

24
test/unit/obj/test_updater.py
View File

@ -53,10 +53,14 @@ class TestObjectUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1) rmtree(self.testdir, ignore_errors=1)
def test_creation(self): def test_creation(self):
cu = object_updater.ObjectUpdater( cu = object_updater.ObjectUpdater({
{'devices': self.devices_dir, 'mount_check': 'false', 'devices': self.devices_dir,
'swift_dir': self.testdir}, 'mount_check': 'false',
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) 'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger')) self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None) self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir) self.assertEquals(cu.devices, self.devices_dir)
@ -66,10 +70,14 @@ class TestObjectUpdater(unittest.TestCase):
self.assert_(cu.get_container_ring() is not None) self.assert_(cu.get_container_ring() is not None)
def test_update_once_single_threaded(self): def test_update_once_single_threaded(self):
cu = object_updater.ObjectUpdater( cu = object_updater.ObjectUpdater({
{'devices': self.devices_dir, 'mount_check': 'false', 'devices': self.devices_dir,
'swift_dir': self.testdir}, 'mount_check': 'false',
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) 'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded() cu.update_once_single_threaded()
async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) async_dir = os.path.join(self.sda1, object_server.ASYNCDIR)
os.mkdir(async_dir) os.mkdir(async_dir)

20
test/unit/proxy/test_server.py
View File

@ -1100,17 +1100,6 @@ class TestObjectController(unittest.TestCase):
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
try: try:
# healthcheck test
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(body, 'OK')
# Check bad version # Check bad version
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
@ -1129,15 +1118,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 404' exp = 'HTTP/1.1 404'
self.assertEquals(headers[:len(exp)], exp) self.assertEquals(headers[:len(exp)], exp)
# Check bad method
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 405'
self.assertEquals(headers[:len(exp)], exp)
# Check blacklist # Check blacklist
prosrv.rate_limit_blacklist = ['a'] prosrv.rate_limit_blacklist = ['a']
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))