diff --git a/bin/swauth-cleanup-tokens b/bin/swauth-cleanup-tokens new file mode 100755 index 0000000000..a24ed2015c --- /dev/null +++ b/bin/swauth-cleanup-tokens @@ -0,0 +1,102 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import simplejson as json +except ImportError: + import json +import re +from datetime import datetime, timedelta +from optparse import OptionParser +from sys import argv, exit +from time import sleep, time + +from swift.common.client import Connection + + +if __name__ == '__main__': + parser = OptionParser(usage='Usage: %prog [options]') + parser.add_option('-t', '--token-life', dest='token_life', + default='86400', help='The expected life of tokens; token objects ' + 'modified more than this number of seconds ago will be checked for ' + 'expiration (default: 86400).') + parser.add_option('-s', '--sleep', dest='sleep', + default='0.1', help='The number of seconds to sleep between token ' + 'checks (default: 0.1)') + parser.add_option('-v', '--verbose', dest='verbose', action='store_true', + default=False, help='Outputs everything done instead of just the ' + 'deletions.') + parser.add_option('-A', '--admin-url', dest='admin_url', + default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' + 'subsystem (default: http://127.0.0.1:8080/auth/)') + parser.add_option('-K', '--admin-key', dest='admin_key', + help='The key for .super_admin.') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + if len(args) != 0: + parser.parse_args(['-h']) + options.admin_url = options.admin_url.rstrip('/') + if not options.admin_url.endswith('/v1.0'): + options.admin_url += '/v1.0' + options.admin_user = '.super_admin:.super_admin' + options.token_life = timedelta(0, float(options.token_life)) + options.sleep = float(options.sleep) + conn = Connection(options.admin_url, options.admin_user, options.admin_key) + for x in xrange(16): + container = '.token_%x' % x + marker = None + while True: + if options.verbose: + print 'GET %s?marker=%s' % (container, marker) + objs = conn.get_container(container, marker=marker)[1] + if objs: + marker = objs[-1]['name'] + else: + if options.verbose: + print 'No more objects in %s' % container + break + for obj in objs: + last_modified = datetime(*map(int, re.split('[^\d]', + obj['last_modified'])[:-1])) + ago = datetime.utcnow() - last_modified + if ago > options.token_life: + if options.verbose: + print '%s/%s last modified %ss ago; investigating' % \ + (container, obj['name'], + ago.days * 86400 + ago.seconds) + print 'GET %s/%s' % (container, obj['name']) + detail = conn.get_object(container, obj['name'])[1] + detail = json.loads(detail) + if detail['expires'] < time(): + if options.verbose: + print '%s/%s expired %ds ago; deleting' % \ + (container, obj['name'], + time() - detail['expires']) + print 'DELETE %s/%s' % (container, obj['name']) + conn.delete_object(container, obj['name']) + elif options.verbose: + print "%s/%s won't expire for %ds; skipping" % \ + (container, obj['name'], + detail['expires'] - time()) + elif options.verbose: + print '%s/%s last modified %ss ago; skipping' % \ + (container, obj['name'], + ago.days * 86400 + ago.seconds) + sleep(options.sleep) + if options.verbose: + print 'Done.' diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index 65ad7d0ca4..068501a0b2 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -232,6 +232,21 @@ get performance timings (warning: the initial populate takes a while). These timings are dumped into a CSV file (/etc/swift/stats.csv by default) and can then be graphed to see how cluster performance is trending. +------------------------------------ +Additional Cleanup Script for Swauth +------------------------------------ + +If you decide to use Swauth, you'll want to install a cronjob to clean up any +orphaned expired tokens. These orphaned tokens can occur when a "stampede" +occurs where a single user authenticates several times concurrently. Generally, +these orphaned tokens don't pose much of an issue, but it's good to clean them +up once a "token life" period (default: 1 day or 86400 seconds). + +This should be as simple as adding `swauth-cleanup-tokens -K swauthkey > +/dev/null` to a crontab entry on one of the proxies that is running Swauth; but +run `swauth-cleanup-tokens` with no arguments for detailed help on the options +available. + ------------------------ Debugging Tips and Tools ------------------------ diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index fe38c02342..2a84357bb9 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -455,6 +455,20 @@ See :ref:`config-proxy` for the initial setup, and then follow these additional #. After you sync all the nodes, make sure the admin has the keys in /etc/swift and the ownership for the ring file is correct. +Additional Cleanup Script for Swauth +------------------------------------ + +If you decide to use Swauth, you'll want to install a cronjob to clean up any +orphaned expired tokens. These orphaned tokens can occur when a "stampede" +occurs where a single user authenticates several times concurrently. Generally, +these orphaned tokens don't pose much of an issue, but it's good to clean them +up once a "token life" period (default: 1 day or 86400 seconds). + +This should be as simple as adding `swauth-cleanup-tokens -K swauthkey > +/dev/null` to a crontab entry on one of the proxies that is running Swauth; but +run `swauth-cleanup-tokens` with no arguments for detailed help on the options +available. + Troubleshooting Notes --------------------- If you see problems, look in var/log/syslog (or messages on some distros). diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst index b3ad2e9db0..14b5c03ef3 100644 --- a/doc/source/overview_auth.rst +++ b/doc/source/overview_auth.rst @@ -78,13 +78,14 @@ objects contain a JSON dictionary of the format:: The `` can only be `plaintext` at this time, and the `` is the plain text password itself. -The `` contains at least two group names. The first is a unique -group name identifying that user and is of the format `:`. The +The `` contains at least two groups. The first is a unique group +identifying that user and it's name is of the format `:`. The second group is the `` itself. Additional groups of `.admin` for account administrators and `.reseller_admin` for reseller administrators may exist. Here's an example user JSON dictionary:: - {"auth": "plaintext:testing", "groups": ["test:tester", "test", ".admin"]} + {"auth": "plaintext:testing", + "groups": ["name": "test:tester", "name": "test", "name": ".admin"]} To map an auth service account to a Swift storage account, the Service Account Id string is stored in the `X-Container-Meta-Account-Id` header for the @@ -147,7 +148,7 @@ Here's an example token object's JSON dictionary:: {"account": "test", "user": "tester", "account_id": "AUTH_8980f74b1cda41e483cbe0a925f448a9", - "groups": ["test:tester", "test", ".admin"], + "groups": ["name": "test:tester", "name": "test", "name": ".admin"], "expires": 1291273147.1624689} To easily map a user to an already issued token, the token name is stored in @@ -156,14 +157,29 @@ the user object's `X-Object-Meta-Auth-Token` header. Here is an example full listing of an :: .account_id - AUTH_4a4e6655-4c8e-4bcb-b73e-0ff1104c4fef - AUTH_5162ec51-f792-4db3-8a35-b3439a1bf6fd - AUTH_8efbea51-9339-42f8-8ac5-f26e1da67eed - .token - AUTH_tk03d8571f735a4ec9abccc704df941c6e - AUTH_tk27cf3f2029b64ec8b56c5d638807b3de - AUTH_tk7594203449754c22a34ac7d910521c2e - AUTH_tk8f2ee54605dd42a8913d244de544d19e + AUTH_2282f516-559f-4966-b239-b5c88829e927 + AUTH_f6f57a3c-33b5-4e85-95a5-a801e67505c8 + AUTH_fea96a36-c177-4ca4-8c7e-b8c715d9d37b + .token_0 + .token_1 + .token_2 + .token_3 + .token_4 + .token_5 + .token_6 + AUTH_tk9d2941b13d524b268367116ef956dee6 + .token_7 + .token_8 + AUTH_tk93627c6324c64f78be746f1e6a4e3f98 + .token_9 + .token_a + .token_b + .token_c + .token_d + .token_e + AUTH_tk0d37d286af2c43ffad06e99112b3ec4e + .token_f + AUTH_tk766bbde93771489982d8dc76979d11cf reseller .services reseller diff --git a/setup.py b/setup.py index e214b1c722..d1466e0449 100644 --- a/setup.py +++ b/setup.py @@ -81,9 +81,9 @@ setup( 'bin/swift-log-stats-collector', 'bin/swift-account-stats-logger', 'bin/swauth-add-account', 'bin/swauth-add-user', - 'bin/swauth-delete-account', 'bin/swauth-delete-user', - 'bin/swauth-list', 'bin/swauth-prep', 'bin/swauth-set-account-service', - 'bin/swift-auth-to-swauth', + 'bin/swauth-cleanup-tokens', 'bin/swauth-delete-account', + 'bin/swauth-delete-user', 'bin/swauth-list', 'bin/swauth-prep', + 'bin/swauth-set-account-service', 'bin/swift-auth-to-swauth', ], entry_points={ 'paste.app_factory': [ diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 610df8b80f..7f985ece96 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -174,7 +174,7 @@ class Swauth(object): if expires < time(): groups = None if not groups: - path = quote('/v1/%s/.token%s/%s' % + path = quote('/v1/%s/.token_%s/%s' % (self.auth_account, token[-1], token)) resp = self.make_request(env, 'GET', path).get_response(self.app) if resp.status_int // 100 != 2: @@ -205,7 +205,8 @@ class Swauth(object): if not account or not account.startswith(self.reseller_prefix): return self.denied_response(req) user_groups = (req.remote_user or '').split(',') - if '.reseller_admin' in user_groups: + if '.reseller_admin' in user_groups and \ + account[len(self.reseller_prefix)].isalnum(): return None if account in user_groups and (req.method != 'PUT' or container): # If the user is admin for the account and is not trying to do an @@ -339,7 +340,7 @@ class Swauth(object): raise Exception('Could not create container: %s %s' % (path, resp.status)) for container in xrange(16): - path = quote('/v1/%s/.token%x' % (self.auth_account, container)) + path = quote('/v1/%s/.token_%x' % (self.auth_account, container)) resp = self.make_request(req.environ, 'PUT', path).get_response(self.app) if resp.status_int // 100 != 2: @@ -852,7 +853,7 @@ class Swauth(object): (path, resp.status)) candidate_token = resp.headers.get('x-object-meta-auth-token') if candidate_token: - path = quote('/v1/%s/.token%s/%s' % + path = quote('/v1/%s/.token_%s/%s' % (self.auth_account, candidate_token[-1], candidate_token)) resp = self.make_request(req.environ, 'DELETE', path).get_response(self.app) @@ -943,6 +944,13 @@ class Swauth(object): return HTTPBadRequest(request=req) if not all((account, user, key)): return HTTPUnauthorized(request=req) + if user == '.super_admin' and key == self.super_admin_key: + token = self.get_itoken(req.environ) + url = '%s/%s.auth' % (self.dsc_url, self.reseller_prefix) + return Response(request=req, + body=json.dumps({'storage': {'default': 'local', 'local': url}}), + headers={'x-auth-token': token, 'x-storage-token': token, + 'x-storage-url': url}) # Authenticate user path = quote('/v1/%s/%s/%s' % (self.auth_account, account, user)) resp = self.make_request(req.environ, 'GET', @@ -959,7 +967,7 @@ class Swauth(object): token = None candidate_token = resp.headers.get('x-object-meta-auth-token') if candidate_token: - path = quote('/v1/%s/.token%s/%s' % + path = quote('/v1/%s/.token_%s/%s' % (self.auth_account, candidate_token[-1], candidate_token)) resp = self.make_request(req.environ, 'GET', path).get_response(self.app) @@ -987,7 +995,7 @@ class Swauth(object): # Generate new token token = '%stk%s' % (self.reseller_prefix, uuid4().hex) # Save token info - path = quote('/v1/%s/.token%s/%s' % + path = quote('/v1/%s/.token_%s/%s' % (self.auth_account, token[-1], token)) resp = self.make_request(req.environ, 'PUT', path, json.dumps({'account': account, 'user': user, @@ -1050,7 +1058,7 @@ class Swauth(object): if expires < time(): groups = None if not groups: - path = quote('/v1/%s/.token%s/%s' % + path = quote('/v1/%s/.token_%s/%s' % (self.auth_account, token[-1], token)) resp = self.make_request(req.environ, 'GET', path).get_response(self.app) @@ -1129,7 +1137,8 @@ class Swauth(object): raise Exception( 'No memcache set up; required for Swauth middleware') memcache_client.set(memcache_key, (self.itoken_expires, - '.auth,.reseller_admin'), timeout=self.token_life) + '.auth,.reseller_admin,%s.auth' % self.reseller_prefix), + timeout=self.token_life) return self.itoken def get_admin_detail(self, req): diff --git a/test/unit/common/middleware/test_swauth.py b/test/unit/common/middleware/test_swauth.py index 6c8c32bddd..10f05a053e 100644 --- a/test/unit/common/middleware/test_swauth.py +++ b/test/unit/common/middleware/test_swauth.py @@ -2882,7 +2882,7 @@ class TestAuth(unittest.TestCase): self.assert_(itk.startswith('AUTH_itk'), itk) expires, groups = fmc.get('AUTH_/auth/%s' % itk) self.assert_(expires > time(), expires) - self.assertEquals(groups, '.auth,.reseller_admin') + self.assertEquals(groups, '.auth,.reseller_admin,AUTH_.auth') def test_get_admin_detail_fail_no_colon(self): self.test_auth.app = FakeApp(iter([]))