Merge from trunk

bin/st

@@ -44,8 +44,6 @@ except:
 try:
     from swift.common.bufferedhttp \
         import BufferedHTTPConnection as HTTPConnection
-    import gettext
-    gettext.install('swift', unicode=1)
 except:
     from httplib import HTTPConnection
 

bin/swift-account-audit

@@ -20,7 +20,6 @@ from urllib import quote
 from hashlib import md5
 import getopt
 from itertools import chain
-import gettext
 
 import simplejson
 from eventlet.greenpool import GreenPool
@@ -325,7 +324,6 @@ class Auditor(object):
 
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     try:
         optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
     except getopt.GetoptError, err:

bin/swift-auth-add-user

@@ -18,13 +18,11 @@ from ConfigParser import ConfigParser
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-import gettext
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     default_conf = '/etc/swift/auth-server.conf'
     parser = OptionParser(
         usage='Usage: %prog [options] <account> <user> <password>')

bin/swift-auth-recreate-accounts

@@ -17,12 +17,10 @@
 from ConfigParser import ConfigParser
 from optparse import OptionParser
 from sys import argv, exit
-import gettext
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     default_conf = '/etc/swift/auth-server.conf'
     parser = OptionParser(usage='Usage: %prog [options]')
     parser.add_option('-c', '--conf', dest='conf', default=default_conf,

bin/swift-auth-update-reseller-prefixes

@@ -16,13 +16,11 @@
 
 from os.path import basename
 from sys import argv, exit
-import gettext
 
 from swift.common.db import get_db_connection
 
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     app = basename(argv[0])
     if len(argv) != 3:
         exit('''

bin/swift-bench

@@ -20,7 +20,6 @@ import sys
 import signal
 import uuid
 from optparse import OptionParser
-import gettext
 
 from swift.common.bench import BenchController
 from swift.common.utils import readconf, NamedLogger
@@ -56,7 +55,6 @@ SAIO_DEFAULTS = {
     }
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     usage = "usage: %prog [OPTIONS] [CONF_FILE]"
     usage += """\n\nConf file with SAIO defaults:
 

bin/swift-drive-audit

@@ -20,7 +20,6 @@ import re
 import subprocess
 import sys
 from ConfigParser import ConfigParser
-import gettext
 
 from swift.common.utils import get_logger
 
@@ -87,7 +86,6 @@ def comment_fstab(mount_point):
     os.rename('/etc/fstab.new', '/etc/fstab')
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     c = ConfigParser()
     try:
         conf_path = sys.argv[1]

bin/swift-get-nodes

@@ -16,14 +16,11 @@
 
 import sys
 import urllib
-import gettext
 
 from swift.common.ring import Ring
 from swift.common.utils import hash_path
 
 
-gettext.install('swift', unicode=1)
-
 if len(sys.argv) < 3 or len(sys.argv) > 5:
     print 'Usage: %s <ring.gz> <account> [<container>] [<object>]' \
       % sys.argv[0]

bin/swift-log-uploader

@@ -15,14 +15,12 @@
 # limitations under the License.
 
 import sys
-import gettext
 
 from swift.stats.log_uploader import LogUploader
 from swift.common.utils import parse_options
 from swift.common import utils
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
     try:
         plugin = options['extra_args'][0]

bin/swift-object-info

@@ -18,14 +18,12 @@ import sys
 import cPickle as pickle
 from datetime import datetime
 from hashlib import md5
-import gettext
 
 from swift.common.ring import Ring
 from swift.obj.server import read_metadata
 from swift.common.utils import hash_path
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     if len(sys.argv) <= 1:
         print "Usage: %s OBJECT_FILE" % sys.argv[0]
         sys.exit(1)

bin/swift-ring-builder

@@ -21,7 +21,6 @@ from os import mkdir
 from os.path import basename, dirname, exists, join as pathjoin
 from sys import argv, exit
 from time import time
-import gettext
 
 from swift.common.ring import RingBuilder
 
@@ -175,7 +174,6 @@ swift-ring-builder <builder_file> set_min_part_hours <hours>
 
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     if len(argv) < 2:
         print '''
 swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s

bin/swift-stats-populate

@@ -21,7 +21,6 @@ from optparse import OptionParser
 from sys import exit, argv
 from time import time
 from uuid import uuid4
-import gettext
 
 from eventlet import GreenPool, patcher, sleep
 from eventlet.pools import Pool
@@ -76,7 +75,6 @@ def report(success):
 
 if __name__ == '__main__':
     global begun, created, item_type, next_report, need_to_create, retries_done
-    gettext.install('swift', unicode=1)
     patcher.monkey_patch()
 
     parser = OptionParser()

bin/swift-stats-report

@@ -23,7 +23,6 @@ from optparse import OptionParser
 from sys import argv, exit, stderr
 from time import time
 from uuid import uuid4
-import gettext
 
 from eventlet import GreenPool, hubs, patcher, sleep, Timeout
 from eventlet.pools import Pool
@@ -747,7 +746,6 @@ def object_delete_report(coropool, connpool, options):
 
 
 if __name__ == '__main__':
-    gettext.install('swift', unicode=1)
     patcher.monkey_patch()
     hubs.get_hub().debug_exceptions = False
 

swift/__init__.py

@@ -1 +1,5 @@
+import gettext
+
+
 __version__ = '1.1.0'
+gettext.install('swift')

swift/common/daemon.py

@@ -16,7 +16,6 @@
 import os
 import sys
 import signal
-import gettext
 from re import sub
 
 from swift.common import utils
@@ -42,7 +41,6 @@ class Daemon(object):
         utils.validate_configuration()
         utils.capture_stdio(self.logger, **kwargs)
         utils.drop_privileges(self.conf.get('user', 'swift'))
-        gettext.install('swift', unicode=1)
 
         def kill_children(*args):
             signal.signal(signal.SIGTERM, signal.SIG_IGN)

swift/common/db.py

@@ -139,6 +139,7 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
         conn.execute('PRAGMA synchronous = NORMAL')
         conn.execute('PRAGMA count_changes = OFF')
         conn.execute('PRAGMA temp_store = MEMORY')
+        conn.execute('PRAGMA journal_mode = DELETE')
         conn.create_function('chexor', 3, chexor)
     except sqlite3.DatabaseError:
         import traceback

swift/common/middleware/memcache.py

@@ -35,6 +35,8 @@ class MemcacheMiddleware(object):
 def filter_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
+
     def cache_filter(app):
         return MemcacheMiddleware(app, conf)
+
     return cache_filter

swift/common/wsgi.py

@@ -21,7 +21,6 @@ import signal
 import sys
 import time
 import mimetools
-import gettext
 
 import eventlet
 from eventlet import greenio, GreenPool, sleep, wsgi, listen
@@ -57,14 +56,15 @@ def monkey_patch_mimetools():
 
     mimetools.Message.parsetype = parsetype
 
+
 def get_socket(conf, default_port=8080):
     """Bind socket to bind ip:port in conf
 
     :param conf: Configuration dict to read settings from
     :param default_port: port to use if not specified in conf
 
-    :returns : a socket object as returned from socket.listen or ssl.wrap_socket
+    :returns : a socket object as returned from socket.listen or
-               if conf specifies cert_file
+               ssl.wrap_socket if conf specifies cert_file
     """
     bind_addr = (conf.get('bind_ip', '0.0.0.0'),
                  int(conf.get('bind_port', default_port)))
@@ -121,7 +121,6 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
     sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
     # remaining tasks should not require elevated privileges
     drop_privileges(conf.get('user', 'swift'))
-    gettext.install('swift', unicode=1)
 
     # finally after binding to ports and privilege drop, run app __init__ code
     app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})

swift/proxy/server.py

@@ -96,9 +96,11 @@ def delay_denial(func):
         return func(*a, **kw)
     return wrapped
 
+
 def get_account_memcache_key(account):
     return 'account/%s' % account
 
+
 def get_container_memcache_key(account, container):
     return 'container/%s/%s' % (account, container)
 
@@ -298,7 +300,8 @@ class Controller(object):
         :param additional_info: additional information to log
         """
         self.app.logger.exception(
-            _('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: %(info)s'),
+            _('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: '
+              '%(info)s'),
             {'type': typ, 'ip': node['ip'], 'port': node['port'],
              'device': node['device'], 'info': additional_info})
 
@@ -1149,19 +1152,17 @@ class ObjectController(Controller):
             return HTTPPreconditionFailed(request=req,
                     body='Destination header must be of the form '
                          '<container name>/<object name>')
-        new_source = '/' + self.container_name + '/' + self.object_name
+        source = '/' + self.container_name + '/' + self.object_name
         self.container_name = dest_container
         self.object_name = dest_object
-        new_headers = {}
+        # re-write the existing request as a PUT instead of creating a new one
-        for k, v in req.headers.items():
+        # since this one is already attached to the posthooklogger
-            new_headers[k] = v
+        req.method = 'PUT'
-        new_headers['X-Copy-From'] = new_source
+        req.path_info = '/' + self.account_name + dest
-        new_headers['Content-Length'] = 0
+        req.headers['Content-Length'] = 0
-        del new_headers['Destination']
+        req.headers['X-Copy-From'] = source
-        new_path = '/' + self.account_name + dest
+        del req.headers['Destination']
-        new_req = Request.blank(new_path, environ=req.environ,
+        return self.PUT(req)
-                                headers=new_headers)
-        return self.PUT(new_req)
 
 
 class ContainerController(Controller):

swift/stats/access_processor.py

@@ -127,7 +127,8 @@ class AccessLogProcessor(object):
         d['code'] = int(d['code'])
         return d
 
-    def process(self, obj_stream, account, container, object_name):
+    def process(self, obj_stream, data_object_account, data_object_container,
+                data_object_name):
         '''generate hourly groupings of data from one access log file'''
         hourly_aggr_info = {}
         total_lines = 0
@@ -191,7 +192,8 @@ class AccessLogProcessor(object):
 
             hourly_aggr_info[aggr_key] = d
         if bad_lines > (total_lines * self.warn_percent):
-            name = '/'.join([account, container, object_name])
+            name = '/'.join([data_object_account, data_object_container,
+                             data_object_name])
             self.logger.warning('I found a bunch of bad lines in %s '\
                         '(%d bad, %d total)' % (name, bad_lines, total_lines))
         return hourly_aggr_info

swift/stats/stats_processor.py

@@ -22,10 +22,11 @@ class StatsLogProcessor(object):
     def __init__(self, conf):
         self.logger = get_logger(conf)
 
-    def process(self, obj_stream, account, container, object_name):
+    def process(self, obj_stream, data_object_account, data_object_container,
+                data_object_name):
         '''generate hourly groupings of data from one stats log file'''
         account_totals = {}
-        year, month, day, hour, _ = object_name.split('/')
+        year, month, day, hour, _ = data_object_name.split('/')
         for line in obj_stream:
             if not line:
                 continue

test/functionalnosetests/test_container.py

@@ -115,6 +115,10 @@ class TestContainer(unittest.TestCase):
         resp.read()
         self.assert_(resp.status in (200, 204), resp.status)
         self.assertEquals(resp.getheader('x-container-meta-test'), 'Value')
+        resp = retry(delete, name)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+
         name = uuid4().hex
         resp = retry(put, name, '')
         resp.read()

test/functionalnosetests/test_object.py

@@ -38,21 +38,110 @@ class TestObject(unittest.TestCase):
         if skip:
             raise SkipTest
 
+        def delete(url, token, parsed, conn, obj):
+            conn.request('DELETE',
+                         '%s/%s/%s' % (parsed.path, self.container, obj),
+                         '', {'X-Auth-Token': token})
+            return check_response(conn)
+
+        # get list of objects in container
+        def list(url, token, parsed, conn):
+            conn.request('GET',
+                         '%s/%s' % (parsed.path, self.container),
+                         '', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(list)
+        object_listing = resp.read()
+        self.assertEquals(resp.status, 200)
+
+        # iterate over object listing and delete all objects
+        for obj in object_listing.splitlines():
+            resp = retry(delete, obj)
+            resp.read()
+            self.assertEquals(resp.status, 204)
+
+        # delete the container
         def delete(url, token, parsed, conn):
-            conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container,
+            conn.request('DELETE', parsed.path + '/' + self.container, '',
-                self.obj), '', {'X-Auth-Token': token})
+                         {'X-Auth-Token': token})
             return check_response(conn)
         resp = retry(delete)
         resp.read()
         self.assertEquals(resp.status, 204)
 
+    def test_copy_object(self):
+        if skip:
+            raise SkipTest
+
+        source = '%s/%s' % (self.container, self.obj)
+        dest = '%s/%s' % (self.container, 'test_copy')
+
+        # get contents of source
+        def get_source(url, token, parsed, conn):
+            conn.request('GET',
+                         '%s/%s' % (parsed.path, source),
+                         '', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get_source)
+        source_contents = resp.read()
+        self.assertEquals(resp.status, 200)
+        self.assertEquals(source_contents, 'test')
+
+        # copy source to dest with X-Copy-From
+        def put(url, token, parsed, conn):
+            conn.request('PUT', '%s/%s' % (parsed.path, dest), '',
+                         {'X-Auth-Token': token,
+                          'Content-Length': '0',
+                          'X-Copy-From': source})
+            return check_response(conn)
+        resp = retry(put)
+        contents = resp.read()
+        self.assertEquals(resp.status, 201)
+
+        # contents of dest should be the same as source
+        def get_dest(url, token, parsed, conn):
+            conn.request('GET',
+                         '%s/%s' % (parsed.path, dest),
+                         '', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get_dest)
+        dest_contents = resp.read()
+        self.assertEquals(resp.status, 200)
+        self.assertEquals(dest_contents, source_contents)
+
+        # delete the copy
         def delete(url, token, parsed, conn):
-            conn.request('DELETE', parsed.path + '/' + self.container, '',
+            conn.request('DELETE', '%s/%s' % (parsed.path, dest), '',
                          {'X-Auth-Token': token})
             return check_response(conn)
         resp = retry(delete)
         resp.read()
         self.assertEquals(resp.status, 204)
+        # verify dest does not exist
+        resp = retry(get_dest)
+        resp.read()
+        self.assertEquals(resp.status, 404)
+
+        # copy source to dest with COPY
+        def copy(url, token, parsed, conn):
+            conn.request('COPY', '%s/%s' % (parsed.path, source), '',
+                         {'X-Auth-Token': token,
+                          'Destination': dest})
+            return check_response(conn)
+        resp = retry(copy)
+        contents = resp.read()
+        self.assertEquals(resp.status, 201)
+
+        # contents of dest should be the same as source
+        resp = retry(get_dest)
+        dest_contents = resp.read()
+        self.assertEquals(resp.status, 200)
+        self.assertEquals(dest_contents, source_contents)
+
+        # delete the copy
+        resp = retry(delete)
+        resp.read()
+        self.assertEquals(resp.status, 204)
 
     def test_public_object(self):
         if skip:

test/unit/container/test_updater.py

@@ -23,6 +23,7 @@ from shutil import rmtree
 from eventlet import spawn, TimeoutError, listen
 from eventlet.timeout import Timeout
 
+from swift.common import utils
 from swift.container import updater as container_updater
 from swift.container import server as container_server
 from swift.common.db import ContainerBroker
@@ -33,6 +34,7 @@ from swift.common.utils import normalize_timestamp
 class TestContainerUpdater(unittest.TestCase):
 
     def setUp(self):
+        utils.HASH_PATH_SUFFIX = 'endcap'
         self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
         if not self.path_to_test_xfs or \
                 not os.path.exists(self.path_to_test_xfs):