Changes to support deployment on IPv6

bin/swauth-add-account

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-add-user

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-delete-account

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-delete-user

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-list

@@ -22,9 +22,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-prep

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-set-account-service

@@ -22,9 +22,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swift-ring-builder

@@ -48,6 +48,8 @@ The <search-value> can be of the form:
         /sdb1            Matches devices with the device name sdb1
         _shiny           Matches devices with shiny in the meta data
         _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
+        [::1]            Matches devices in any zone with the ip ::1
+        z1-[::1]:5678    Matches devices in zone 1 with the ip ::1 and port 5678
     Most specific example:
         d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
     Nerd explanation:
@@ -76,6 +78,13 @@ The <search-value> can be of the form:
             i += 1
         match.append(('ip', search_value[:i]))
         search_value = search_value[i:]
+    elif len(search_value) and search_value[0] == '[':
+        i = 1
+        while i < len(search_value) and search_value[i] != ']':
+            i += 1
+        i += 1
+        match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
+        search_value = search_value[i:]
     if search_value.startswith(':'):
         i = 1
         while i < len(search_value) and search_value[i].isdigit():
@@ -110,6 +119,16 @@ The <search-value> can be of the form:
     return devs
 
 
+def format_device(dev):
+    """
+    Format a device for display.
+    """
+    if ':' in dev['ip']:
+        return 'd%(id)sz%(zone)s-[%(ip)s]:%(port)s/%(device)s_"%(meta)s"' % dev
+    else:
+        return 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
+
+
 class Commands:
 
     def unknown():
@@ -235,6 +254,14 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
             print 'Invalid add value: %s' % argv[3]
             exit(EXIT_ERROR)
         i = 1
+        if rest[i] == '[':
+            i += 1
+            while i < len(rest) and rest[i] != ']':
+                i += 1
+            i += 1
+            ip = rest[1:i].lstrip('[').rstrip(']')
+            rest = rest[i:]
+        else:
             while i < len(rest) and rest[i] in '0123456789.':
                 i += 1
             ip = rest[1:i]
@@ -279,6 +306,10 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
         builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip,
                          'port': port, 'device': device_name, 'weight': weight,
                          'meta': meta})
+        if ':' in ip:
+            print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \
+                  (zone, ip, port, device_name, meta, weight, next_dev_id)
+        else:
             print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \
                   (zone, ip, port, device_name, meta, weight, next_dev_id)
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
@@ -342,6 +373,13 @@ swift-ring-builder <builder_file> set_info <search-value>
                 i += 1
             change.append(('ip', change_value[:i]))
             change_value = change_value[i:]
+        elif len(change_value) and change_value[0] == '[':
+            i = 1
+            while i < len(change_value) and change_value[i] != ']':
+                i += 1
+            i += 1
+            change.append(('ip', change_value[:i].lstrip('[').rstrip(']')))
+            change_value = change_value[i:]
         if change_value.startswith(':'):
             i = 1
             while i < len(change_value) and change_value[i].isdigit():
@@ -366,15 +404,13 @@ swift-ring-builder <builder_file> set_info <search-value>
         if len(devs) > 1:
             print 'Matched more than one device:'
             for dev in devs:
-                print '    d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \
+                print '    %s' % format_device(dev)
-                      '"%(meta)s"' % dev
             if raw_input('Are you sure you want to update the info for '
                          'these %s devices? (y/N) ' % len(devs)) != 'y':
                 print 'Aborting device modifications'
                 exit(EXIT_ERROR)
         for dev in devs:
-            orig_dev_string = \
+            orig_dev_string = format_device(dev)
-                'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
             test_dev = dict(dev)
             for key, value in change:
                 test_dev[key] = value
@@ -390,9 +426,7 @@ swift-ring-builder <builder_file> set_info <search-value>
                     exit(EXIT_ERROR)
             for key, value in change:
                 dev[key] = value
-            new_dev_string = \
+            print 'Device %s is now %s' % (orig_dev_string, format_device(dev))
-                'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
-            print 'Device %s is now %s' % (orig_dev_string, new_dev_string)
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 

swift/auth/server.py

@@ -20,7 +20,6 @@ from contextlib import contextmanager
 from time import gmtime, strftime, time
 from urllib import unquote, quote
 from uuid import uuid4
-from urlparse import urlparse
 from hashlib import md5, sha1
 import hmac
 import base64
@@ -32,7 +31,7 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 from swift.common.db import get_db_connection
-from swift.common.utils import get_logger, split_path
+from swift.common.utils import get_logger, split_path, urlparse
 
 
 class AuthController(object):

swift/common/bench.py

@@ -16,13 +16,12 @@
 import uuid
 import time
 import random
-from urlparse import urlparse
 from contextlib import contextmanager
 
 import eventlet.pools
 from eventlet.green.httplib import CannotSendRequest
 
-from swift.common.utils import TRUE_VALUES
+from swift.common.utils import TRUE_VALUES, urlparse
 from swift.common import client
 from swift.common import direct_client
 

swift/common/middleware/acl.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from urlparse import urlparse
+from swift.common.utils import urlparse
 
 
 def clean_acl(name, value):

swift/common/middleware/swauth.py

@@ -21,7 +21,6 @@ from httplib import HTTPConnection, HTTPSConnection
 from time import gmtime, strftime, time
 from traceback import format_exc
 from urllib import quote, unquote
-from urlparse import urlparse
 from uuid import uuid4
 from hashlib import md5, sha1
 import hmac
@@ -36,7 +35,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
-from swift.common.utils import cache_from_env, get_logger, split_path
+from swift.common.utils import cache_from_env, get_logger, split_path, urlparse
 
 
 class Swauth(object):

swift/common/utils.py

@@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError
 from optparse import OptionParser
 from tempfile import mkstemp
 import cPickle as pickle
+from urlparse import urlparse as stdlib_urlparse, ParseResult
 
 import eventlet
 from eventlet import greenio, GreenPool, sleep, Timeout, listen
@@ -861,3 +862,35 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5):
     elif running_time - now > time_per_request:
         eventlet.sleep((running_time - now) / clock_accuracy)
     return running_time + time_per_request
+
+
+class ModifiedParseResult(ParseResult):
+    "Parse results class for urlparse."
+
+    @property
+    def hostname(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        if netloc.startswith('['):
+            return netloc[1:].split(']')[0]
+        elif ':' in netloc:
+            return netloc.rsplit(':')[0]
+        return netloc
+
+    @property
+    def port(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        if netloc.startswith('['):
+            netloc = netloc.rsplit(']')[1]
+        if ':' in netloc:
+            return int(netloc.rsplit(':')[1])
+        return None
+
+
+def urlparse(url):
+    """
+    urlparse augmentation.
+    This is necessary because urlparse can't handle RFC 2732 URLs.
+
+    :param url: URL to parse.
+    """
+    return ModifiedParseResult(*stdlib_urlparse(url))

swift/common/wsgi.py

@@ -68,11 +68,15 @@ def get_socket(conf, default_port=8080):
     """
     bind_addr = (conf.get('bind_ip', '0.0.0.0'),
                  int(conf.get('bind_port', default_port)))
+    address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0],
+            bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)
+            if addr[0] in (socket.AF_INET, socket.AF_INET6)][0]
     sock = None
     retry_until = time.time() + 30
     while not sock and time.time() < retry_until:
         try:
-            sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)))
+            sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)),
+                        family=address_family)
             if 'cert_file' in conf:
                 sock = ssl.wrap_socket(sock, certfile=conf['cert_file'],
                     keyfile=conf['key_file'])

swift/container/server.py

@@ -89,7 +89,7 @@ class ContainerController(object):
         account_partition = req.headers.get('X-Account-Partition')
         account_device = req.headers.get('X-Account-Device')
         if all([account_host, account_partition, account_device]):
-            account_ip, account_port = account_host.split(':')
+            account_ip, account_port = account_host.rsplit(':', 1)
             new_path = '/' + '/'.join([account, container])
             info = broker.get_info()
             account_headers = {'x-put-timestamp': info['put_timestamp'],

swift/obj/server.py

@@ -301,7 +301,7 @@ class ObjectController(object):
         full_path = '/%s/%s/%s' % (account, container, obj)
         try:
             with ConnectionTimeout(self.conn_timeout):
-                ip, port = host.split(':')
+                ip, port = host.rsplit(':', 1)
                 conn = http_connect(ip, port, contdevice, partition, op,
                         full_path, headers_out)
             with Timeout(self.node_timeout):

test/unit/common/test_utils.py

@@ -626,6 +626,28 @@ log_name = yarr'''
             total += i
         self.assertTrue(abs(50 - (time.time() - start) * 100) < 10)
 
+    def test_urlparse(self):
+        parsed = utils.urlparse('http://127.0.0.1/')
+        self.assertEquals(parsed.scheme, 'http')
+        self.assertEquals(parsed.hostname, '127.0.0.1')
+        self.assertEquals(parsed.path, '/')
+
+        parsed = utils.urlparse('http://127.0.0.1:8080/')
+        self.assertEquals(parsed.port, 8080)
+
+        parsed = utils.urlparse('https://127.0.0.1/')
+        self.assertEquals(parsed.scheme, 'https')
+
+        parsed = utils.urlparse('http://[::1]/')
+        self.assertEquals(parsed.hostname, '::1')
+
+        parsed = utils.urlparse('http://[::1]:8080/')
+        self.assertEquals(parsed.hostname, '::1')
+        self.assertEquals(parsed.port, 8080)
+
+        parsed = utils.urlparse('www.example.com')
+        self.assertEquals(parsed.hostname, '')
+
     def test_ratelimit_sleep_with_sleep(self):
         running_time = 0
         start = time.time()