From 419c2cb95f5ce0c515fd8636d90065dfcf784c8c Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 20 Jun 2011 17:56:20 -0700 Subject: [PATCH 001/334] initial --- README.rst | 31 +++++++++++++ keystonelight/__init__.py | 0 keystonelight/backends/__init__.py | 0 keystonelight/backends/pam.py | 21 +++++++++ keystonelight/identity.py | 28 ++++++++++++ keystonelight/keystone_compat.py | 38 ++++++++++++++++ keystonelight/service.py | 70 ++++++++++++++++++++++++++++++ keystonelight/token.py | 16 +++++++ keystonelight/utils.py | 39 +++++++++++++++++ tools/pip-requires | 2 + 10 files changed, 245 insertions(+) create mode 100644 README.rst create mode 100644 keystonelight/__init__.py create mode 100644 keystonelight/backends/__init__.py create mode 100644 keystonelight/backends/pam.py create mode 100644 keystonelight/identity.py create mode 100644 keystonelight/keystone_compat.py create mode 100644 keystonelight/service.py create mode 100644 keystonelight/token.py create mode 100644 keystonelight/utils.py create mode 100644 tools/pip-requires diff --git a/README.rst b/README.rst new file mode 100644 index 0000000000..c6b57552f1 --- /dev/null +++ b/README.rst @@ -0,0 +1,31 @@ + + +General expected data model: + + ( tenants >--< users ) --< roles + + Tenants and Users have a many-to-many relationship. + A given Tenant-User pair can have many Roles. + + +Tenant Schema: + id: something big and unique + name: something displayable + .. created_at: datetime + .. deleted_at: datetime + +User Schema: + id: something big and unique + name: something displayable + .. created_at: datetime + .. deleted_at: datetime + + +General service model: + + (1) a web service with an API + (2) a variety of backend storage schemes for tenant-user pairs. + (3) a simple token datastore + + + diff --git a/keystonelight/__init__.py b/keystonelight/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystonelight/backends/__init__.py b/keystonelight/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystonelight/backends/pam.py b/keystonelight/backends/pam.py new file mode 100644 index 0000000000..8896c93049 --- /dev/null +++ b/keystonelight/backends/pam.py @@ -0,0 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from __future__ import absolute_imports + +import pam + + +class PamIdentity(object): + """Very basic identity based on PAM. + + Tenant is always the same as User, root user has admin role. + """ + + def authenticate(self, username, password): + if pam.authenticate(username, password): + extras = {} + if username == 'root': + extras['is_admin'] == True + # NOTE(termie): (tenant, user, extras) + return (username, username, extras) + diff --git a/keystonelight/identity.py b/keystonelight/identity.py new file mode 100644 index 0000000000..ee2345aa50 --- /dev/null +++ b/keystonelight/identity.py @@ -0,0 +1,28 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# these will be the basic data types for tenants and users +# backends will make use of them to return something that conforms to their apis + + +import hflags as flags + +from keystonelight import utils + + +FLAGS = flags.FLAGS +flags.DEFINE_string('identity_driver', + 'keystonelight.backends.dummy.DummyIdentity', + 'identity driver to handle identity requests') + + +class IdentityManager(object): + def __init__(self): + self.driver = utils.import_object(FLAGS.identity_driver) + + + def authenticate(self, context, **kwargs): + """Passthru authentication to the identity driver. + + This call will basically just result in getting a token. + """ + return self.driver.authenticate(**kwargs) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py new file mode 100644 index 0000000000..680c9d93f9 --- /dev/null +++ b/keystonelight/keystone_compat.py @@ -0,0 +1,38 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# this is the web service frontend that emulates keystone +from keystonelight import service + +def _token_to_keystone(token): + return {'id': token['id'], + 'expires': token.get('expires', '') + + +class KeystoneIdentityController(service.IdentityController): + def authenticate(self, context, **kwargs): + token = super(KeystoneIdentityController, self).authenticate( + context, **kwargs) + return {'auth': {'token': _token_to_keystone(token), + 'serviceCatalog': SERVICE_CATALOG}} + + +class KeystoneTokenController(service.TokenController): + def validate_token(self, context, token_id): + token = super(KeystoneTokenController, self).validate_token( + context, token_id) + # TODO(termie): munge into keystone format + + tenants = [{'tenantId': token['tenant']['id'], + 'name': token['tenant']['name']}] + roles = [] + if token['extras'].get('is_admin'): + roles.append({ + 'id': 1, + 'href': 'https://.openstack.org/identity/v2.0/roles/admin', + 'tenantId': token['tenant']['id']}) + + return {'auth': {'token': _token_to_keystone(token), + 'user': {'groups': {'group': tenants}, + 'roleRefs': {'roleRef': roles} + 'username': token['user']['name'], + 'tenantId': token['tenant']['id']}}} diff --git a/keystonelight/service.py b/keystonelight/service.py new file mode 100644 index 0000000000..e436cebc53 --- /dev/null +++ b/keystonelight/service.py @@ -0,0 +1,70 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# this is the web service frontend + +import hflags as flags + +from keystonelight import wsgi + + +FLAGS = flags.FLAGS + + +class TokenController(wsgi.Controller): + """Validate and pass through calls to TokenManager.""" + + def __init__(self): + self.token_api = token.Manager() + + def validate_token(self, context, token_id): + token = self.validate_token(context, token_id) + return token + + +class IdentityController(wsgi.Controller): + """Validate and pass calls through to IdentityManager. + + IdentityManager will also pretty much just pass calls through to + a specific driver. + """ + + def __init__(self): + self.identity_api = identity.Manager() + self.token_api = token.Manager() + + def authenticate(self, context, **kwargs): + tenant, user, extras = self.identity_api.authenticate(context, **kwargs) + token = self.token_api.create_token(context, + tenant=tenant, + user=user, + extras=extras) + return token + + + +class Router(object): + def __init__(self): + token_controller = TokenController() + identity_controller = IdentityController() + + mapper.connect('/v2.0/token', controller=identity_controller, + action='authenticate') + mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + action='revoke_token', + conditions=dict(method=['DELETE'])) + + +class AdminRouter(object): + def __init__(self): + token_controller = TokenController() + identity_controller = IdentityController() + + mapper.connect('/v2.0/token', controller=identity_controller, + action='authenticate') + mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + action='validate_token', + conditions=dict(method=['GET'])) + mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + action='revoke_token', + conditions=dict(method=['DELETE'])) + diff --git a/keystonelight/token.py b/keystonelight/token.py new file mode 100644 index 0000000000..ffdd4c4c54 --- /dev/null +++ b/keystonelight/token.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# the token interfaces + +from keystonelight import identity + +class TokenManager(object): + def create_token(self, context, data): + pass + + def validate_token(self, context, token_id): + """Return info for a token if it is valid.""" + pass + + def revoke_token(self, context, token_id): + pass diff --git a/keystonelight/utils.py b/keystonelight/utils.py new file mode 100644 index 0000000000..406ad19e15 --- /dev/null +++ b/keystonelight/utils.py @@ -0,0 +1,39 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def import_class(import_str): + """Returns a class from a string including module and class.""" + mod_str, _sep, class_str = import_str.rpartition('.') + try: + __import__(mod_str) + return getattr(sys.modules[mod_str], class_str) + except (ImportError, ValueError, AttributeError), exc: + LOG.debug(_('Inner Exception: %s'), exc) + raise exception.ClassNotFound(class_name=class_str) + + +def import_object(import_str): + """Returns an object including a module or module and class.""" + try: + __import__(import_str) + return sys.modules[import_str] + except ImportError: + cls = import_class(import_str) + return cls() diff --git a/tools/pip-requires b/tools/pip-requires new file mode 100644 index 0000000000..14d62412c0 --- /dev/null +++ b/tools/pip-requires @@ -0,0 +1,2 @@ +hflags +pam==0.1.4 From 158dfbac2aa91a8c0287f4398e11bc90f7d84ac0 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 20 Jun 2011 18:37:51 -0700 Subject: [PATCH 002/334] most bits working --- bin/keystone | 40 ++++ keystonelight/backends/pam.py | 2 +- keystonelight/identity.py | 4 +- keystonelight/service.py | 98 ++++++++-- keystonelight/token.py | 14 +- keystonelight/utils.py | 7 +- keystonelight/wsgi.py | 358 ++++++++++++++++++++++++++++++++++ tools/pip-requires | 5 + 8 files changed, 505 insertions(+), 23 deletions(-) create mode 100755 bin/keystone create mode 100644 keystonelight/wsgi.py diff --git a/bin/keystone b/bin/keystone new file mode 100755 index 0000000000..a3309f6d15 --- /dev/null +++ b/bin/keystone @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import os +import sys + +# If ../../keystone/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'keystonelight', '__init__.py')): + sys.path.insert(0, possible_topdir) + +import logging + +import hflags as flags + +from keystonelight import service +from keystonelight import wsgi + + +FLAGS = flags.FLAGS + +flags.DEFINE_boolean('verbose', True, 'verbose logging') + +if __name__ == '__main__': + args = FLAGS(sys.argv) + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + public = service.Router() + admin = service.AdminRouter() + + public = service.PostParamsMiddleware(public) + + server = wsgi.Server() + server.start(public, 8080) + server.start(admin, 8081) + server.wait() diff --git a/keystonelight/backends/pam.py b/keystonelight/backends/pam.py index 8896c93049..cfac3ed508 100644 --- a/keystonelight/backends/pam.py +++ b/keystonelight/backends/pam.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -from __future__ import absolute_imports +from __future__ import absolute_import import pam diff --git a/keystonelight/identity.py b/keystonelight/identity.py index ee2345aa50..8d6c7b6e37 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -11,11 +11,11 @@ from keystonelight import utils FLAGS = flags.FLAGS flags.DEFINE_string('identity_driver', - 'keystonelight.backends.dummy.DummyIdentity', + 'keystonelight.backends.pam.PamIdentity', 'identity driver to handle identity requests') -class IdentityManager(object): +class Manager(object): def __init__(self): self.driver = utils.import_object(FLAGS.identity_driver) diff --git a/keystonelight/service.py b/keystonelight/service.py index e436cebc53..a06f99cbf4 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -2,26 +2,92 @@ # this is the web service frontend -import hflags as flags +import json +import logging +import hflags as flags +import routes +import webob.dec + +from keystonelight import identity +from keystonelight import token +from keystonelight import utils from keystonelight import wsgi FLAGS = flags.FLAGS +# TODO(termie): these should probably be paste configs instead +flags.DEFINE_string('token_controller', + 'keystonelight.service.TokenController', + 'token controller') +flags.DEFINE_string('identity_controller', + 'keystonelight.service.IdentityController', + 'identity controller') -class TokenController(wsgi.Controller): + +class BaseApplication(wsgi.Application): + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + del arg_dict['controller'] + logging.info('arg_dict: %s', arg_dict) + + context = req.environ.get('openstack.context', {}) + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + params.update(arg_dict) + + # TODO(termie): do some basic normalization on methods + method = getattr(self, action) + + # NOTE(vish): make sure we have no unicode keys for py2.6. + params = dict([(str(k), v) for (k, v) in params.iteritems()]) + result = method(context, **params) + + if result is None or type(result) is str or type(result) is unicode: + return result + + return json.dumps(result) + + +class PostParamsMiddleware(wsgi.Middleware): + """Middleware to allow method arguments to be passed as POST parameters. + + Filters out the parameters `self`, `context` and anything beginning with + an underscore. + + """ + + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + +class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" def __init__(self): self.token_api = token.Manager() def validate_token(self, context, token_id): - token = self.validate_token(context, token_id) + token = self.token_api.validate_token(context, token_id) return token -class IdentityController(wsgi.Controller): +class IdentityController(BaseApplication): """Validate and pass calls through to IdentityManager. IdentityManager will also pretty much just pass calls through to @@ -35,29 +101,33 @@ class IdentityController(wsgi.Controller): def authenticate(self, context, **kwargs): tenant, user, extras = self.identity_api.authenticate(context, **kwargs) token = self.token_api.create_token(context, - tenant=tenant, - user=user, - extras=extras) + dict(tenant=tenant, + user=user, + extras=extras)) + logging.info(token) return token -class Router(object): +class Router(wsgi.Router): def __init__(self): - token_controller = TokenController() - identity_controller = IdentityController() + token_controller = utils.import_object(FLAGS.token_controller) + identity_controller = utils.import_object(FLAGS.identity_controller) + mapper = routes.Mapper() mapper.connect('/v2.0/token', controller=identity_controller, action='authenticate') mapper.connect('/v2.0/token/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) + super(Router, self).__init__(mapper) -class AdminRouter(object): +class AdminRouter(wsgi.Router): def __init__(self): - token_controller = TokenController() - identity_controller = IdentityController() + token_controller = utils.import_object(FLAGS.token_controller) + identity_controller = utils.import_object(FLAGS.identity_controller) + mapper = routes.Mapper() mapper.connect('/v2.0/token', controller=identity_controller, action='authenticate') @@ -67,4 +137,4 @@ class AdminRouter(object): mapper.connect('/v2.0/token/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) - + super(AdminRouter, self).__init__(mapper) diff --git a/keystonelight/token.py b/keystonelight/token.py index ffdd4c4c54..5221fe9eb1 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -2,15 +2,21 @@ # the token interfaces +import uuid + from keystonelight import identity -class TokenManager(object): +STORE = {} + +class Manager(object): def create_token(self, context, data): - pass + token = uuid.uuid4().hex + STORE[token] = data + return token def validate_token(self, context, token_id): """Return info for a token if it is valid.""" - pass + return STORE.get(token_id) def revoke_token(self, context, token_id): - pass + STORE.pop(token_id) diff --git a/keystonelight/utils.py b/keystonelight/utils.py index 406ad19e15..96f9185efd 100644 --- a/keystonelight/utils.py +++ b/keystonelight/utils.py @@ -17,6 +17,9 @@ # License for the specific language governing permissions and limitations # under the License. +import logging +import sys + def import_class(import_str): """Returns a class from a string including module and class.""" @@ -25,8 +28,8 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError), exc: - LOG.debug(_('Inner Exception: %s'), exc) - raise exception.ClassNotFound(class_name=class_str) + logging.debug('Inner Exception: %s', exc) + raise def import_object(import_str): diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py new file mode 100644 index 0000000000..5eea49085a --- /dev/null +++ b/keystonelight/wsgi.py @@ -0,0 +1,358 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import logging +import os +import sys + +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True, time=True) +import hflags as flags +import routes +import routes.middleware +import webob +import webob.dec +import webob.exc +from paste import deploy + + +FLAGS = flags.FLAGS + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.DEBUG): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + +class Server(object): + """Server class to manage multiple WSGI sockets and applications.""" + + def __init__(self, threads=1000): + self.pool = eventlet.GreenPool(threads) + self.socket_info = {} + + def start(self, application, port, host='0.0.0.0', key=None, backlog=128): + """Run a WSGI server with the given application.""" + arg0 = sys.argv[0] + logging.debug('Starting %(arg0)s on %(host)s:%(port)s' % locals()) + socket = eventlet.listen((host, port), backlog=backlog) + self.pool.spawn_n(self._run, application, socket) + if key: + self.socket_info[key] = socket.getsockname() + + def wait(self): + """Wait until all servers have completed running.""" + try: + self.pool.waitall() + except KeyboardInterrupt: + pass + + def _run(self, application, socket): + """Start a WSGI server in a new green thread.""" + logger = logging.getLogger('eventlet.wsgi.server') + eventlet.wsgi.server(socket, application, custom_pool=self.pool, + log=WritableLogger(logger)) + + +class Request(webob.Request): + pass + + +class Application(object): + """Base WSGI application wrapper. Subclasses need to implement __call__.""" + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [app:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [app:wadl] + latest_version = 1.3 + paste.app_factory = nova.api.fancy_api:Wadl.factory + + which would result in a call to the `Wadl` class as + + import nova.api.fancy_api + fancy_api.Wadl(latest_version='1.3') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + return cls(**local_config) + + def __call__(self, environ, start_response): + r"""Subclasses will probably want to implement __call__ like this: + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + # Any of the following objects work as responses: + + # Option 1: simple string + res = 'message\n' + + # Option 2: a nicely formatted HTTP exception page + res = exc.HTTPForbidden(detail='Nice try') + + # Option 3: a webob Response object (in case you need to play with + # headers, or you want to be treated like an iterable, or or or) + res = Response(); + res.app_iter = open('somefile') + + # Option 4: any wsgi app to be run next + res = self.application + + # Option 5: you can get a Response object for a wsgi app, too, to + # play with headers etc + res = req.get_response(self.application) + + # You can then just return your response... + return res + # ... or set req.response and return None. + req.response = res + + See the end of http://pythonpaste.org/webob/modules/dec.html + for more info. + + """ + raise NotImplementedError('You must implement __call__') + + +class Middleware(Application): + """Base WSGI middleware. + + These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + + """ + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = nova.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import nova.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + return cls(app, **local_config) + return _factory + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) + + +class Debug(Middleware): + """Helper class for debugging a WSGI application. + + Can be inserted into any WSGI application chain to get information + about the request and response. + + """ + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + print ('*' * 40) + ' REQUEST ENVIRON' + for key, value in req.environ.items(): + print key, '=', value + print + resp = req.get_response(self.application) + + print ('*' * 40) + ' RESPONSE HEADERS' + for (key, value) in resp.headers.iteritems(): + print key, '=', value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """Iterator that prints the contents of a wrapper string.""" + print ('*' * 40) + ' BODY' + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + + +class Router(object): + """WSGI middleware that maps incoming requests to WSGI apps.""" + + def __init__(self, mapper): + """Create a router for the given routes.Mapper. + + Each route in `mapper` must specify a 'controller', which is a + WSGI app to call. You'll probably want to specify an 'action' as + well and have your controller be an object that can route + the request to the action-specific method. + + Examples: + mapper = routes.Mapper() + sc = ServerController() + + # Explicit mapping of one route to a controller+action + mapper.connect(None, '/svrlist', controller=sc, action='list') + + # Actions are all implicitly defined + mapper.resource('server', 'servers', controller=sc) + + # Pointing to an arbitrary WSGI app. You can specify the + # {path_info:.*} parameter so the target app can be handed just that + # section of the URL. + mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp()) + + """ + self.map = mapper + self._router = routes.middleware.RoutesMiddleware(self._dispatch, + self.map) + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + """Route the incoming request to a controller based on self.map. + + If no match, return a 404. + + """ + return self._router + + @staticmethod + @webob.dec.wsgify(RequestClass=Request) + def _dispatch(req): + """Dispatch the request to the appropriate controller. + + Called by self._router after matching the incoming request to a route + and putting the information into req.environ. Either returns 404 + or the routed WSGI app's response. + + """ + match = req.environ['wsgiorg.routing_args'][1] + if not match: + return webob.exc.HTTPNotFound() + app = match['controller'] + return app + + +def paste_config_file(basename): + """Find the best location in the system for a paste config file. + + Search Order + ------------ + + The search for a paste config file honors `FLAGS.state_path`, which in a + version checked out from bzr will be the `nova` directory in the top level + of the checkout, and in an installation for a package for your distribution + will likely point to someplace like /etc/nova. + + This method tries to load places likely to be used in development or + experimentation before falling back to the system-wide configuration + in `/etc/nova/`. + + * Current working directory + * the `etc` directory under state_path, because when working on a checkout + from bzr this will point to the default + * top level of FLAGS.state_path, for distributions + * /etc/nova, which may not be diffrerent from state_path on your distro + + """ + configfiles = [basename, + os.path.join(FLAGS.state_path, 'etc', 'nova', basename), + os.path.join(FLAGS.state_path, 'etc', basename), + os.path.join(FLAGS.state_path, basename), + '/etc/nova/%s' % basename] + for configfile in configfiles: + if os.path.exists(configfile): + return configfile + + +def load_paste_configuration(filename, appname): + """Returns a paste configuration dict, or None.""" + filename = os.path.abspath(filename) + config = None + try: + config = deploy.appconfig('config:%s' % filename, name=appname) + except LookupError: + pass + return config + + +def load_paste_app(filename, appname): + """Builds a wsgi app from a paste config, None if app not configured.""" + filename = os.path.abspath(filename) + app = None + try: + app = deploy.loadapp('config:%s' % filename, name=appname) + except LookupError: + pass + return app diff --git a/tools/pip-requires b/tools/pip-requires index 14d62412c0..30a8d58337 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,2 +1,7 @@ hflags pam==0.1.4 +WebOb==0.9.8 +eventlet==0.9.12 +PasteDeploy +paste +routes From 9a0ec99e2da07863cb4ea032a1b0c6b5d959a149 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 21 Jun 2011 02:08:15 +0000 Subject: [PATCH 003/334] rudimentary login working --- bin/keystone | 3 +- keystonelight/backends/pam.py | 11 ++++++-- keystonelight/keystone_compat.py | 17 ++++++++---- keystonelight/service.py | 47 ++++++++++++++++++++++++++------ 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/bin/keystone b/bin/keystone index a3309f6d15..fa75bc01ce 100755 --- a/bin/keystone +++ b/bin/keystone @@ -23,6 +23,7 @@ from keystonelight import wsgi FLAGS = flags.FLAGS flags.DEFINE_boolean('verbose', True, 'verbose logging') +flags.DEFINE_flag(flags.HelpFlag()) if __name__ == '__main__': args = FLAGS(sys.argv) @@ -32,7 +33,7 @@ if __name__ == '__main__': public = service.Router() admin = service.AdminRouter() - public = service.PostParamsMiddleware(public) + public = service.JsonBodyMiddleware(public) server = wsgi.Server() server.start(public, 8080) diff --git a/keystonelight/backends/pam.py b/keystonelight/backends/pam.py index cfac3ed508..f50e1abbb5 100644 --- a/keystonelight/backends/pam.py +++ b/keystonelight/backends/pam.py @@ -11,11 +11,16 @@ class PamIdentity(object): Tenant is always the same as User, root user has admin role. """ - def authenticate(self, username, password): + def authenticate(self, username, password, **kwargs): if pam.authenticate(username, password): extras = {} if username == 'root': extras['is_admin'] == True - # NOTE(termie): (tenant, user, extras) - return (username, username, extras) + + tenant = {'id': username, + 'name': username} + user = {'id': username, + 'name': username} + + return (tenant, user, extras) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 680c9d93f9..7fc566469f 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -1,15 +1,22 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # this is the web service frontend that emulates keystone +import logging + from keystonelight import service def _token_to_keystone(token): - return {'id': token['id'], - 'expires': token.get('expires', '') + return {'id': token, + 'expires': ''} + + +SERVICE_CATALOG = {"cdn": [{"adminURL": "http://cdn.admin-nets.local/v1.1/1234", "region": "RegionOne", "internalURL": "http://33.33.33.10:7777/v1.1/1234", "publicURL": "http://cdn.publicinternets.com/v1.1/1234"}], "nova_compat": [{"adminURL": "http://33.33.33.10:8774/v1.0", "region": "RegionOne", "internalURL": "http://33.33.33.10:8774/v1.0", "publicURL": "http://nova.publicinternets.com/v1.0/"}], "nova": [{"adminURL": "http://33.33.33.10:8774/v1.1", "region": "RegionOne", "internalURL": "http://33.33.33.10:8774/v1.1", "publicURL": "http://nova.publicinternets.com/v1.1/"}], "keystone": [{"adminURL": "http://33.33.33.10:8081/v2.0", "region": "RegionOne", "internalURL": "http://33.33.33.10:8080/v2.0", "publicURL": "http://keystone.publicinternets.com/v2.0"}], "glance": [{"adminURL": "http://nova.admin-nets.local/v1.1/1234", "region": "RegionOne", "internalURL": "http://33.33.33.10:9292/v1.1/1234", "publicURL": "http://glance.publicinternets.com/v1.1/1234"}], "swift": [{"adminURL": "http://swift.admin-nets.local:8080/", "region": "RegionOne", "internalURL": "http://33.33.33.10:8080/v1/AUTH_1234", "publicURL": "http://swift.publicinternets.com/v1/AUTH_1234"}]} + class KeystoneIdentityController(service.IdentityController): def authenticate(self, context, **kwargs): + kwargs = kwargs['passwordCredentials'] token = super(KeystoneIdentityController, self).authenticate( context, **kwargs) return {'auth': {'token': _token_to_keystone(token), @@ -27,12 +34,12 @@ class KeystoneTokenController(service.TokenController): roles = [] if token['extras'].get('is_admin'): roles.append({ - 'id': 1, - 'href': 'https://.openstack.org/identity/v2.0/roles/admin', + 'roleId': 'Admin', + 'href': 'https://www.openstack.org/identity/v2.0/roles/admin', 'tenantId': token['tenant']['id']}) return {'auth': {'token': _token_to_keystone(token), 'user': {'groups': {'group': tenants}, - 'roleRefs': {'roleRef': roles} + 'roleRefs': roles, 'username': token['user']['name'], 'tenantId': token['tenant']['id']}}} diff --git a/keystonelight/service.py b/keystonelight/service.py index a06f99cbf4..758b44c94e 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -76,6 +76,38 @@ class PostParamsMiddleware(wsgi.Middleware): request.environ['openstack.params'] = params +class JsonBodyMiddleware(wsgi.Middleware): + """Middleware to allow method arguments to be passed as serialized JSON. + + Accepting arguments as JSON is useful for accepting data that may be more + complex than simple primitives. + + In this case we accept it as urlencoded data under the key 'json' as in + json= but this could be extended to accept raw JSON + in the POST body. + + Filters out the parameters `self`, `context` and anything beginning with + an underscore. + + """ + + def process_request(self, request): + #if 'json' not in request.params: + # return + + params_json = request.body + params_parsed = json.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" @@ -83,8 +115,8 @@ class TokenController(BaseApplication): self.token_api = token.Manager() def validate_token(self, context, token_id): - token = self.token_api.validate_token(context, token_id) - return token + token_info = self.token_api.validate_token(context, token_id) + return token_info class IdentityController(BaseApplication): @@ -108,16 +140,15 @@ class IdentityController(BaseApplication): return token - class Router(wsgi.Router): def __init__(self): token_controller = utils.import_object(FLAGS.token_controller) identity_controller = utils.import_object(FLAGS.identity_controller) mapper = routes.Mapper() - mapper.connect('/v2.0/token', controller=identity_controller, + mapper.connect('/v2.0/tokens', controller=identity_controller, action='authenticate') - mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) super(Router, self).__init__(mapper) @@ -129,12 +160,12 @@ class AdminRouter(wsgi.Router): identity_controller = utils.import_object(FLAGS.identity_controller) mapper = routes.Mapper() - mapper.connect('/v2.0/token', controller=identity_controller, + mapper.connect('/v2.0/tokens', controller=identity_controller, action='authenticate') - mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, action='validate_token', conditions=dict(method=['GET'])) - mapper.connect('/v2.0/token/{token_id}', controller=token_controller, + mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) super(AdminRouter, self).__init__(mapper) From 8cd7f5c8108c8911007fe1c1e9a37e23f21b0459 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 20 Jun 2011 19:27:23 -0700 Subject: [PATCH 004/334] add get_tenants --- bin/keystone | 1 + keystonelight/backends/pam.py | 3 +++ keystonelight/identity.py | 3 +++ keystonelight/service.py | 17 +++++++++++++++++ 4 files changed, 24 insertions(+) diff --git a/bin/keystone b/bin/keystone index fa75bc01ce..ce361eaef7 100755 --- a/bin/keystone +++ b/bin/keystone @@ -33,6 +33,7 @@ if __name__ == '__main__': public = service.Router() admin = service.AdminRouter() + public = service.TokenAuthMiddleware(public) public = service.JsonBodyMiddleware(public) server = wsgi.Server() diff --git a/keystonelight/backends/pam.py b/keystonelight/backends/pam.py index f50e1abbb5..a685bd13db 100644 --- a/keystonelight/backends/pam.py +++ b/keystonelight/backends/pam.py @@ -24,3 +24,6 @@ class PamIdentity(object): return (tenant, user, extras) + def get_tenants(self, username): + return [{'id': username, + 'name': username}] diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 8d6c7b6e37..c8e525db1d 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -26,3 +26,6 @@ class Manager(object): This call will basically just result in getting a token. """ return self.driver.authenticate(**kwargs) + + def get_tenants(self, context, user_id): + return self.driver.get_tenants(user_id) diff --git a/keystonelight/service.py b/keystonelight/service.py index 758b44c94e..42cdfa29e0 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -55,6 +55,13 @@ class BaseApplication(wsgi.Application): return json.dumps(result) +class TokenAuthMiddleware(wsgi.Middleware): + def process_request(self, request): + token = requeset.headers.get('X-Auth-Token') + logging.info('GOT TOKEN %s', token) + request.environ['openstack.context'] = {'token': token} + + class PostParamsMiddleware(wsgi.Middleware): """Middleware to allow method arguments to be passed as POST parameters. @@ -139,6 +146,12 @@ class IdentityController(BaseApplication): logging.info(token) return token + def get_tenants(self, context): + token_id = context.get('token') + token = self.token_api.validate_token(context, token_id) + + return self.identity_api.get_tenants(context, + user_id=token['user']['id']) class Router(wsgi.Router): def __init__(self): @@ -151,6 +164,10 @@ class Router(wsgi.Router): mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) + + mapper.connect("/v2.0/tenants", controller=identity_controller, + action="get_tenants", conditions=dict(method=["GET"])) + super(Router, self).__init__(mapper) From a328b99178f794fa84aba7127230b230189a260a Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 21 Jun 2011 02:32:38 +0000 Subject: [PATCH 005/334] working with dashboard --- bin/keystone | 2 ++ keystonelight/keystone_compat.py | 6 ++++++ keystonelight/service.py | 11 ++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/bin/keystone b/bin/keystone index ce361eaef7..fcfa378a93 100755 --- a/bin/keystone +++ b/bin/keystone @@ -36,6 +36,8 @@ if __name__ == '__main__': public = service.TokenAuthMiddleware(public) public = service.JsonBodyMiddleware(public) + admin = service.TokenAuthMiddleware(admin) + server = wsgi.Server() server.start(public, 8080) server.start(admin, 8081) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 7fc566469f..b282909c31 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -22,6 +22,12 @@ class KeystoneIdentityController(service.IdentityController): return {'auth': {'token': _token_to_keystone(token), 'serviceCatalog': SERVICE_CATALOG}} + def get_tenants(self, context): + tenants = super(KeystoneIdentityController, self).get_tenants(context) + return {'tenants': {'values': [{'id': x['id'], 'description': x['name'], 'enabled': True} + for x in tenants]}} + + class KeystoneTokenController(service.TokenController): def validate_token(self, context, token_id): diff --git a/keystonelight/service.py b/keystonelight/service.py index 42cdfa29e0..7424cf1dce 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -8,6 +8,7 @@ import logging import hflags as flags import routes import webob.dec +import webob.exc from keystonelight import identity from keystonelight import token @@ -57,7 +58,8 @@ class BaseApplication(wsgi.Application): class TokenAuthMiddleware(wsgi.Middleware): def process_request(self, request): - token = requeset.headers.get('X-Auth-Token') + logging.info('GOT HEADERS %s', request.headers) + token = request.headers.get('X-Auth-Token') logging.info('GOT TOKEN %s', token) request.environ['openstack.context'] = {'token': token} @@ -103,6 +105,9 @@ class JsonBodyMiddleware(wsgi.Middleware): # return params_json = request.body + if not params_json: + return + params_parsed = json.loads(params_json) params = {} for k, v in params_parsed.iteritems(): @@ -123,6 +128,8 @@ class TokenController(BaseApplication): def validate_token(self, context, token_id): token_info = self.token_api.validate_token(context, token_id) + if not token_info: + raise webob.exc.HTTPUnauthorized() return token_info @@ -148,11 +155,13 @@ class IdentityController(BaseApplication): def get_tenants(self, context): token_id = context.get('token') + logging.info("GET TENANTS %s", token_id) token = self.token_api.validate_token(context, token_id) return self.identity_api.get_tenants(context, user_id=token['user']['id']) + class Router(wsgi.Router): def __init__(self): token_controller = utils.import_object(FLAGS.token_controller) From 03b75a5e696c1d7423b498f0dbc4752d29aa4dd0 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 11 Oct 2011 14:55:25 -0700 Subject: [PATCH 006/334] added a test, need to get it working now --- keystonelight/backends/kvs.py | 30 ++++++++++++ keystonelight/identity.py | 1 - keystonelight/models.py | 17 +++++++ keystonelight/test.py | 20 ++++++++ keystonelight/utils.py | 39 +++++++++++++++ tests/test_keystone_compat.py | 90 +++++++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 keystonelight/backends/kvs.py create mode 100644 keystonelight/models.py create mode 100644 keystonelight/test.py create mode 100644 tests/test_keystone_compat.py diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py new file mode 100644 index 0000000000..117c71f358 --- /dev/null +++ b/keystonelight/backends/kvs.py @@ -0,0 +1,30 @@ +class DictKvs(dict): + def set(self, key, value): + return self[key] = value + + +class KvsIdentity(object): + def __init__(self, db=None): + if db is None: + db = DictKvs() + self.db = db + + # Public Interface + def tenants_for_token(self, token_id): + token = self.db.get('token-%s' % token_id) + user = self.db.get('user-%s' % token['user']) + o = [] + for tenant_id in user['tenants']: + o.append(self.db.get('tenant-%s' % tenant_id)) + + return o + + # Private CRUD for testing + def _create_user(self, id, user): + self.db.set('user-%s' % id, user) + + def _create_tenant(self, id, tenant): + self.db.set('tenant-%s' % id, tenant) + + def _create_token(self, id, token): + self.db.set('token-%s' % id, token) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index c8e525db1d..c067cb331b 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -19,7 +19,6 @@ class Manager(object): def __init__(self): self.driver = utils.import_object(FLAGS.identity_driver) - def authenticate(self, context, **kwargs): """Passthru authentication to the identity driver. diff --git a/keystonelight/models.py b/keystonelight/models.py new file mode 100644 index 0000000000..a56e8061ad --- /dev/null +++ b/keystonelight/models.py @@ -0,0 +1,17 @@ + + +class Token(dict): + def __init__(self, id=None, user=None, tenant=None, *args, **kw): + super(Token, self).__init__(id=id, user=user, tenant=tenant, *args, **kw) + + +class User(dict): + def __init__(self, id=None, tenants=None, *args, **kw): + if tenants is None: + tenants = [] + super(User, self).__init__(id=id, tenants=[], *args, **kw) + + +class Tenant(dict): + def __init__(self, id=None, *args, **kw): + super(Tenant, self).__init__(id=id, *args, **kw) diff --git a/keystonelight/test.py b/keystonelight/test.py new file mode 100644 index 0000000000..801bae1ce1 --- /dev/null +++ b/keystonelight/test.py @@ -0,0 +1,20 @@ +import os +import unittest + + +ROOTDIR = os.path.dirname(os.path.dirname(__file__)) +VENDOR = os.path.join(ROOTDIR, 'vendor') + + +class TestCase(unittest.TestCase): + def assertDictEquals(self, expected, actual): + for k in expected: + self.assertTrue(k in actual, + "Expected key %s not in %s." % (k, actual)) + self.assertEquals(expected[k], actual[k], + "Expected value for %s to be '%s', not '%s'." + % (k, expected[k], actual[k])) + for k in actual: + self.assertTrue(k in expected, + "Unexpected key %s in %s." % (k, actual)) + diff --git a/keystonelight/utils.py b/keystonelight/utils.py index 96f9185efd..2f7088276d 100644 --- a/keystonelight/utils.py +++ b/keystonelight/utils.py @@ -18,6 +18,7 @@ # under the License. import logging +import subprocess import sys @@ -40,3 +41,41 @@ def import_object(import_str): except ImportError: cls = import_class(import_str) return cls() + +# From python 2.7 +def check_output(*popenargs, **kwargs): + r"""Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + logging.debug(' '.join(popenargs[0])) + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output + + +def git(*args): + return check_output(['git'] + list(args)) diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py new file mode 100644 index 0000000000..f372f9a4cb --- /dev/null +++ b/tests/test_keystone_compat.py @@ -0,0 +1,90 @@ +import copy +import os +import json + +from keystonelight import models +from keystonelight import test +from keystonelight import utils + + +IDENTITY_API_REPO = 'git://github.com/openstack/identity-api.git' + + +SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' + + +cd = os.chdir + + +def checkout_samples(rev): + """Make sure we have a checkout of the API docs.""" + revdir = os.path.join(test.VENDOR, 'identity-api-%s' % rev) + + if not os.path.exists(revdir): + utils.git('clone', IDENTITY_API_REPO, revdir) + + cd(revdir) + utils.git('pull') + utils.git('checkout', rev) + return revdir + + +class CompatTestCase(test.TestCase): + def setUp(self): + super(CompatTestCase, self).setUp() + + self.auth_creds = json.load(open( + os.path.join(self.sampledir, 'auth_credentials.json'))) + self.auth_creds_notenant = copy.deepcopy(self.auth_creds) + self.auth_creds_notenant['auth'].pop('tenantName', None) + + self.tenants_for_token = json.load(open( + os.path.join(self.sampledir, 'tenants.json'))) + + # For the tenants for token call + self.user_foo = self.backend._create_user( + 'foo', + models.User(id='foo', tenants=['1234', '3456'])) + self.tenant_1234 = self.backend._create_tenant( + '1234', + models.Tenant(id='1234', + name='ACME Corp', + description='A description...', + enabled=True)) + self.tenant_3456 = self.backend._create_tenant( + '3456', + models.Tenant(id='3456', + name='Iron Works', + description='A description...', + enabled=True)) + + self.token_foo_unscoped = self.backend._create_token( + 'foo_unscoped', + models.Token(id='foo_unscoped', + user='foo')) + self.token_foo_scoped = self.backend._create_token( + 'foo_scoped', + models.Token(id='foo_unscoped', + user='foo', + tenant='1234')) + + +class HeadCompatTestCase(CompatTestCase): + def setUp(self): + revdir = checkout_samples('HEAD') + self.sampledir = os.path.join(revdir, SAMPLE_DIR) + super(HeadCompatTestCase, self).setUp() + + def test_tenants_for_token_unscoped(self): + # get_tenants_for_token + client = self.api.client(token=self.token_foo_unscoped['id']) + resp = client.get('/v2.0/tenants') + data = json.loads(resp.body) + self.assertDictEquals(self.tenants_for_token, data) + + def test_tenants_for_token_scoped(self): + # get_tenants_for_token + client = self.api.client(token=self.token_foo_scoped['id']) + resp = client.get('/v2.0/tenants') + data = json.loads(resp.body) + self.assertDictEquals(self.tenants_for_token, data) From a200e5007e0b0ba5b88a3555dea5600a1915701c Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 11 Oct 2011 15:11:27 -0700 Subject: [PATCH 007/334] add a test client --- keystonelight/test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/keystonelight/test.py b/keystonelight/test.py index 801bae1ce1..5b5e89d102 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -1,10 +1,37 @@ import os import unittest +from keystonelight import wsgi + ROOTDIR = os.path.dirname(os.path.dirname(__file__)) VENDOR = os.path.join(ROOTDIR, 'vendor') +class TestClient(object): + def __init__(self, endpoint=None, token=None): + self.endpoint = None + self.token = token + + def request(self, method, path, headers=None, body=None): + if headers is None: + headers = {} + req = wsgi.Request.blank(path) + req.method = method + for k, v in headers.iteritems(): + req.headers[k] = v + if req.body: + req.body = body + return req.get_response(self.endpoint) + + def get(self, path, headers=None): + return self.request('GET', path=path, headers=headers) + + def post(self, path, headers=None, body=None): + return self.request('POST', path=path, headers=headers, body=body) + + def put(self, path, headers=None, body=None): + return self.request('PUT', path=path, headers=headers, body=body) + class TestCase(unittest.TestCase): def assertDictEquals(self, expected, actual): From 35ec29740681f0e016f66a1be82aa5469db6c59b Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 11 Oct 2011 15:57:07 -0700 Subject: [PATCH 008/334] tests running through, still failing --- keystonelight/backends/kvs.py | 11 +++++-- keystonelight/identity.py | 14 +++----- keystonelight/keystone_compat.py | 3 +- keystonelight/service.py | 56 +++++++++++++++++--------------- keystonelight/test.py | 18 ++++++++-- keystonelight/token.py | 3 ++ keystonelight/utils.py | 4 +-- tests/keystone_compat_HEAD.conf | 5 +++ tests/test_keystone_compat.py | 9 +++-- 9 files changed, 76 insertions(+), 47 deletions(-) create mode 100644 tests/keystone_compat_HEAD.conf diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 117c71f358..55289051f1 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -1,12 +1,14 @@ + class DictKvs(dict): def set(self, key, value): - return self[key] = value + self[key] = value +INMEMDB = DictKvs() class KvsIdentity(object): - def __init__(self, db=None): + def __init__(self, options, db=None): if db is None: - db = DictKvs() + db = INMEMDB self.db = db # Public Interface @@ -22,9 +24,12 @@ class KvsIdentity(object): # Private CRUD for testing def _create_user(self, id, user): self.db.set('user-%s' % id, user) + return user def _create_tenant(self, id, tenant): self.db.set('tenant-%s' % id, tenant) + return tenant def _create_token(self, id, token): self.db.set('token-%s' % id, token) + return token diff --git a/keystonelight/identity.py b/keystonelight/identity.py index c067cb331b..9b887038da 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -4,20 +4,14 @@ # backends will make use of them to return something that conforms to their apis -import hflags as flags - from keystonelight import utils -FLAGS = flags.FLAGS -flags.DEFINE_string('identity_driver', - 'keystonelight.backends.pam.PamIdentity', - 'identity driver to handle identity requests') - - class Manager(object): - def __init__(self): - self.driver = utils.import_object(FLAGS.identity_driver) + def __init__(self, options): + self.driver = utils.import_object(options['identity_driver'], + options=options) + self.options = options def authenticate(self, context, **kwargs): """Passthru authentication to the identity driver. diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index b282909c31..e500e7d04b 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -4,6 +4,7 @@ import logging from keystonelight import service +from keystonelight import wsgi def _token_to_keystone(token): return {'id': token, @@ -28,7 +29,6 @@ class KeystoneIdentityController(service.IdentityController): for x in tenants]}} - class KeystoneTokenController(service.TokenController): def validate_token(self, context, token_id): token = super(KeystoneTokenController, self).validate_token( @@ -49,3 +49,4 @@ class KeystoneTokenController(service.TokenController): 'roleRefs': roles, 'username': token['user']['name'], 'tenantId': token['tenant']['id']}}} + diff --git a/keystonelight/service.py b/keystonelight/service.py index 7424cf1dce..935b50e537 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -5,7 +5,6 @@ import json import logging -import hflags as flags import routes import webob.dec import webob.exc @@ -16,17 +15,6 @@ from keystonelight import utils from keystonelight import wsgi -FLAGS = flags.FLAGS - -# TODO(termie): these should probably be paste configs instead -flags.DEFINE_string('token_controller', - 'keystonelight.service.TokenController', - 'token controller') -flags.DEFINE_string('identity_controller', - 'keystonelight.service.IdentityController', - 'identity controller') - - class BaseApplication(wsgi.Application): @webob.dec.wsgify def __call__(self, req): @@ -123,8 +111,9 @@ class JsonBodyMiddleware(wsgi.Middleware): class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" - def __init__(self): - self.token_api = token.Manager() + def __init__(self, options): + self.token_api = token.Manager(options=options) + self.options = options def validate_token(self, context, token_id): token_info = self.token_api.validate_token(context, token_id) @@ -140,9 +129,10 @@ class IdentityController(BaseApplication): a specific driver. """ - def __init__(self): - self.identity_api = identity.Manager() - self.token_api = token.Manager() + def __init__(self, options): + self.identity_api = identity.Manager(options=options) + self.token_api = token.Manager(options=options) + self.options = options def authenticate(self, context, **kwargs): tenant, user, extras = self.identity_api.authenticate(context, **kwargs) @@ -163,27 +153,34 @@ class IdentityController(BaseApplication): class Router(wsgi.Router): - def __init__(self): - token_controller = utils.import_object(FLAGS.token_controller) - identity_controller = utils.import_object(FLAGS.identity_controller) + def __init__(self, options): + self.options = options + token_controller = utils.import_object( + options['token_controller'], + options=options) + identity_controller = utils.import_object( + options['identity_controller'], + options=options) mapper = routes.Mapper() - mapper.connect('/v2.0/tokens', controller=identity_controller, action='authenticate') mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) - mapper.connect("/v2.0/tenants", controller=identity_controller, action="get_tenants", conditions=dict(method=["GET"])) - super(Router, self).__init__(mapper) class AdminRouter(wsgi.Router): - def __init__(self): - token_controller = utils.import_object(FLAGS.token_controller) - identity_controller = utils.import_object(FLAGS.identity_controller) + def __init__(self, options): + self.options = options + token_controller = utils.import_object( + options['token_controller'], + options=options) + identity_controller = utils.import_object( + options['identity_controller'], + options=options) mapper = routes.Mapper() mapper.connect('/v2.0/tokens', controller=identity_controller, @@ -195,3 +192,10 @@ class AdminRouter(wsgi.Router): action='revoke_token', conditions=dict(method=['DELETE'])) super(AdminRouter, self).__init__(mapper) + + +def identity_app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return Router(conf) + diff --git a/keystonelight/test.py b/keystonelight/test.py index 5b5e89d102..cb56f2c324 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -1,15 +1,19 @@ import os import unittest +from paste import deploy + from keystonelight import wsgi ROOTDIR = os.path.dirname(os.path.dirname(__file__)) VENDOR = os.path.join(ROOTDIR, 'vendor') +TESTSDIR = os.path.join(ROOTDIR, 'tests') + class TestClient(object): - def __init__(self, endpoint=None, token=None): - self.endpoint = None + def __init__(self, app=None, token=None): + self.app = app self.token = token def request(self, method, path, headers=None, body=None): @@ -21,7 +25,7 @@ class TestClient(object): req.headers[k] = v if req.body: req.body = body - return req.get_response(self.endpoint) + return req.get_response(self.app) def get(self, path, headers=None): return self.request('GET', path=path, headers=headers) @@ -34,6 +38,14 @@ class TestClient(object): class TestCase(unittest.TestCase): + def loadapp(self, config): + if not config.startswith('config:'): + config = 'config:%s.conf' % os.path.join(TESTSDIR, config) + return deploy.loadapp(config) + + def client(self, app, *args, **kw): + return TestClient(app, *args, **kw) + def assertDictEquals(self, expected, actual): for k in expected: self.assertTrue(k in actual, diff --git a/keystonelight/token.py b/keystonelight/token.py index 5221fe9eb1..e29d637d34 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -9,6 +9,9 @@ from keystonelight import identity STORE = {} class Manager(object): + def __init__(self, options): + self.options = options + def create_token(self, context, data): token = uuid.uuid4().hex STORE[token] = data diff --git a/keystonelight/utils.py b/keystonelight/utils.py index 2f7088276d..fcc37f9a43 100644 --- a/keystonelight/utils.py +++ b/keystonelight/utils.py @@ -33,14 +33,14 @@ def import_class(import_str): raise -def import_object(import_str): +def import_object(import_str, *args, **kw): """Returns an object including a module or module and class.""" try: __import__(import_str) return sys.modules[import_str] except ImportError: cls = import_class(import_str) - return cls() + return cls(*args, **kw) # From python 2.7 def check_output(*popenargs, **kwargs): diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_HEAD.conf new file mode 100644 index 0000000000..5976c0b1e3 --- /dev/null +++ b/tests/keystone_compat_HEAD.conf @@ -0,0 +1,5 @@ +[app:main] +token_controller = keystonelight.keystone_compat.KeystoneTokenController +identity_controller = keystonelight.keystone_compat.KeystoneIdentityController +identity_driver = keystonelight.backends.kvs.KvsIdentity +paste.app_factory = keystonelight.service:identity_app_factory diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index f372f9a4cb..f3bb1d3fe1 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -73,18 +73,23 @@ class HeadCompatTestCase(CompatTestCase): def setUp(self): revdir = checkout_samples('HEAD') self.sampledir = os.path.join(revdir, SAMPLE_DIR) + self.app = self.loadapp('keystone_compat_HEAD') + + self.backend = utils.import_object( + self.app.options['identity_driver'], options=self.app.options) + super(HeadCompatTestCase, self).setUp() def test_tenants_for_token_unscoped(self): # get_tenants_for_token - client = self.api.client(token=self.token_foo_unscoped['id']) + client = self.client(self.app, token=self.token_foo_unscoped['id']) resp = client.get('/v2.0/tenants') data = json.loads(resp.body) self.assertDictEquals(self.tenants_for_token, data) def test_tenants_for_token_scoped(self): # get_tenants_for_token - client = self.api.client(token=self.token_foo_scoped['id']) + client = self.client(self.app, token=self.token_foo_scoped['id']) resp = client.get('/v2.0/tenants') data = json.loads(resp.body) self.assertDictEquals(self.tenants_for_token, data) From 50d64c3e07255563c44cdc50a4b5a27cbdeeedcf Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 11 Oct 2011 16:26:55 -0700 Subject: [PATCH 009/334] getting closer, need to match api now --- keystonelight/backends/kvs.py | 30 +++++++++++++++++++++++------- keystonelight/keystone_compat.py | 3 ++- keystonelight/service.py | 2 +- keystonelight/token.py | 14 ++++++++------ tests/keystone_compat_HEAD.conf | 1 + tests/test_keystone_compat.py | 14 ++++++++------ 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 55289051f1..a3d8b73f4f 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -3,6 +3,9 @@ class DictKvs(dict): def set(self, key, value): self[key] = value + def delete(self, key): + del self[key] + INMEMDB = DictKvs() class KvsIdentity(object): @@ -11,10 +14,9 @@ class KvsIdentity(object): db = INMEMDB self.db = db - # Public Interface - def tenants_for_token(self, token_id): - token = self.db.get('token-%s' % token_id) - user = self.db.get('user-%s' % token['user']) + # Public interface + def tenants_for_user(self, user_id): + user = self.db.get('user-%s' % user_id) o = [] for tenant_id in user['tenants']: o.append(self.db.get('tenant-%s' % tenant_id)) @@ -30,6 +32,20 @@ class KvsIdentity(object): self.db.set('tenant-%s' % id, tenant) return tenant - def _create_token(self, id, token): - self.db.set('token-%s' % id, token) - return token + +class KvsToken(object): + def __init__(self, options, db=None): + if db is None: + db = INMEMDB + self.db = db + + # Public interface + def get_token(self, id): + return self.db.get('token-%s' % id) + + def create_token(self, id, data): + self.db.set('token-%s' % id, data) + return data + + def delete_token(self, id): + return self.db.delete('token-%s' % id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index e500e7d04b..bccf51d67f 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -25,7 +25,8 @@ class KeystoneIdentityController(service.IdentityController): def get_tenants(self, context): tenants = super(KeystoneIdentityController, self).get_tenants(context) - return {'tenants': {'values': [{'id': x['id'], 'description': x['name'], 'enabled': True} + return {'tenants': {'values': [{'id': x['id'], + 'description': x['name'], 'enabled': True} for x in tenants]}} diff --git a/keystonelight/service.py b/keystonelight/service.py index 935b50e537..a4075fd077 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -149,7 +149,7 @@ class IdentityController(BaseApplication): token = self.token_api.validate_token(context, token_id) return self.identity_api.get_tenants(context, - user_id=token['user']['id']) + user_id=token['user']) class Router(wsgi.Router): diff --git a/keystonelight/token.py b/keystonelight/token.py index e29d637d34..f5e262ffd8 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -4,22 +4,24 @@ import uuid -from keystonelight import identity +from keystonelight import utils -STORE = {} class Manager(object): def __init__(self, options): self.options = options + self.driver = utils.import_object(options['token_driver'], + options=options) def create_token(self, context, data): token = uuid.uuid4().hex - STORE[token] = data - return token + data['id'] = token + token_ref = self.driver.create_token(token, data) + return token_ref def validate_token(self, context, token_id): """Return info for a token if it is valid.""" - return STORE.get(token_id) + return self.driver.get_token(token_id) def revoke_token(self, context, token_id): - STORE.pop(token_id) + self.driver.delete_token(token_id) diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_HEAD.conf index 5976c0b1e3..9154af1a16 100644 --- a/tests/keystone_compat_HEAD.conf +++ b/tests/keystone_compat_HEAD.conf @@ -2,4 +2,5 @@ token_controller = keystonelight.keystone_compat.KeystoneTokenController identity_controller = keystonelight.keystone_compat.KeystoneIdentityController identity_driver = keystonelight.backends.kvs.KvsIdentity +token_driver = keystonelight.backends.kvs.KvsToken paste.app_factory = keystonelight.service:identity_app_factory diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index f3bb1d3fe1..95822d8f58 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -42,27 +42,27 @@ class CompatTestCase(test.TestCase): os.path.join(self.sampledir, 'tenants.json'))) # For the tenants for token call - self.user_foo = self.backend._create_user( + self.user_foo = self.identity_backend._create_user( 'foo', models.User(id='foo', tenants=['1234', '3456'])) - self.tenant_1234 = self.backend._create_tenant( + self.tenant_1234 = self.identity_backend._create_tenant( '1234', models.Tenant(id='1234', name='ACME Corp', description='A description...', enabled=True)) - self.tenant_3456 = self.backend._create_tenant( + self.tenant_3456 = self.identity_backend._create_tenant( '3456', models.Tenant(id='3456', name='Iron Works', description='A description...', enabled=True)) - self.token_foo_unscoped = self.backend._create_token( + self.token_foo_unscoped = self.token_backend.create_token( 'foo_unscoped', models.Token(id='foo_unscoped', user='foo')) - self.token_foo_scoped = self.backend._create_token( + self.token_foo_scoped = self.token_backend.create_token( 'foo_scoped', models.Token(id='foo_unscoped', user='foo', @@ -75,8 +75,10 @@ class HeadCompatTestCase(CompatTestCase): self.sampledir = os.path.join(revdir, SAMPLE_DIR) self.app = self.loadapp('keystone_compat_HEAD') - self.backend = utils.import_object( + self.identity_backend = utils.import_object( self.app.options['identity_driver'], options=self.app.options) + self.token_backend = utils.import_object( + self.app.options['token_driver'], options=self.app.options) super(HeadCompatTestCase, self).setUp() From c8d4e885df91c5c222b69d21ccb8d58bb1003f06 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 12:00:19 +0300 Subject: [PATCH 010/334] added sequence diagrams for keystone compat --- keystone_compat_flows.sdx | 99 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 keystone_compat_flows.sdx diff --git a/keystone_compat_flows.sdx b/keystone_compat_flows.sdx new file mode 100644 index 0000000000..f1fcc5f02f --- /dev/null +++ b/keystone_compat_flows.sdx @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7427b1a2e20598a192d32188394194b4751cd1e4 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 12:01:58 +0300 Subject: [PATCH 011/334] refactor keystone compat and add catalog service --- keystonelight/backends/kvs.py | 11 +++ keystonelight/catalog.py | 18 ++++ keystonelight/keystone_compat.py | 163 ++++++++++++++++++++++++------- keystonelight/service.py | 4 +- keystonelight/token.py | 4 +- tests/keystone_compat_HEAD.conf | 5 +- 6 files changed, 165 insertions(+), 40 deletions(-) create mode 100644 keystonelight/catalog.py diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index a3d8b73f4f..807492e05b 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -49,3 +49,14 @@ class KvsToken(object): def delete_token(self, id): return self.db.delete('token-%s' % id) + + +class KvsCatalog(object): + def __init__(self, options, db=None): + if db is None: + db = INMEMDB + self.db = db + + # Public interface + def get_catalog(self, user, tenant, extras=None): + return self.db.get('catalog-%s' % tenant['id']) diff --git a/keystonelight/catalog.py b/keystonelight/catalog.py new file mode 100644 index 0000000000..99b870205d --- /dev/null +++ b/keystonelight/catalog.py @@ -0,0 +1,18 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# the catalog interfaces + +import uuid + +from keystonelight import utils + + +class Manager(object): + def __init__(self, options): + self.options = options + self.driver = utils.import_object(options['catalog_driver'], + options=options) + + def get_catalog(self, user, tenant, extras=None): + """Return info for a catalog if it is valid.""" + return self.driver.get_catalog(user, tenant, extras=extras) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index bccf51d67f..98c789528e 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -3,51 +3,146 @@ # this is the web service frontend that emulates keystone import logging +import routes + +from keystonelight import catalog +from keystonelight import identity from keystonelight import service +from keystonelight import token from keystonelight import wsgi -def _token_to_keystone(token): - return {'id': token, - 'expires': ''} + +class KeystoneRouter(wsgi.Router): + def __init__(self, options): + self.options = options + self.keystone_controller = KeystoneController(options) -SERVICE_CATALOG = {"cdn": [{"adminURL": "http://cdn.admin-nets.local/v1.1/1234", "region": "RegionOne", "internalURL": "http://33.33.33.10:7777/v1.1/1234", "publicURL": "http://cdn.publicinternets.com/v1.1/1234"}], "nova_compat": [{"adminURL": "http://33.33.33.10:8774/v1.0", "region": "RegionOne", "internalURL": "http://33.33.33.10:8774/v1.0", "publicURL": "http://nova.publicinternets.com/v1.0/"}], "nova": [{"adminURL": "http://33.33.33.10:8774/v1.1", "region": "RegionOne", "internalURL": "http://33.33.33.10:8774/v1.1", "publicURL": "http://nova.publicinternets.com/v1.1/"}], "keystone": [{"adminURL": "http://33.33.33.10:8081/v2.0", "region": "RegionOne", "internalURL": "http://33.33.33.10:8080/v2.0", "publicURL": "http://keystone.publicinternets.com/v2.0"}], "glance": [{"adminURL": "http://nova.admin-nets.local/v1.1/1234", "region": "RegionOne", "internalURL": "http://33.33.33.10:9292/v1.1/1234", "publicURL": "http://glance.publicinternets.com/v1.1/1234"}], "swift": [{"adminURL": "http://swift.admin-nets.local:8080/", "region": "RegionOne", "internalURL": "http://33.33.33.10:8080/v1/AUTH_1234", "publicURL": "http://swift.publicinternets.com/v1/AUTH_1234"}]} + mapper = routes.Mapper() + mapper.connect('/v2.0/tokens', + controller=self.keystone_controller, + action='authenticate', + conditions=dict(method=['POST'])) + mapper.connect('/v2.0/tokens/{token_id}', + controller=self.keystone_controller, + action='validate_token', + conditions=dict(method=['GET'])) + mapper.connect('/v2.0/tenants', + controller=self.keystone_controller, + action='tenants_for_token', + conditions=dict(method=['GET'])) + super(KeystoneRouter, self).__init__(mapper) +class KeystoneController(service.BaseApplication): + def __init__(self, options): + self.options = options + self.catalog_api = catalog.Manager(options) + self.identity_api = identity.Manager(options) + self.token_api = token.Manager(options) + pass -class KeystoneIdentityController(service.IdentityController): - def authenticate(self, context, **kwargs): - kwargs = kwargs['passwordCredentials'] - token = super(KeystoneIdentityController, self).authenticate( - context, **kwargs) - return {'auth': {'token': _token_to_keystone(token), - 'serviceCatalog': SERVICE_CATALOG}} + def authenticate(self, context, auth=None): + """Authenticate credentials and return a token. - def get_tenants(self, context): - tenants = super(KeystoneIdentityController, self).get_tenants(context) - return {'tenants': {'values': [{'id': x['id'], - 'description': x['name'], 'enabled': True} - for x in tenants]}} + Keystone accepts auth as a dict that looks like: + + { + "auth":{ + "passwordCredentials":{ + "username":"test_user", + "password":"mypass" + }, + "tenantName":"customer-x" + } + } + + In this case, tenant is optional, if not provided the token will be + considered "unscoped" and can later be used to get a scoped token. + + Alternatively, this call accepts auth with only a token and tenant + that will return a token that is scoped to that tenant. + """ + + if 'passwordCredentials' in auth: + username = auth['passwordCredentials'].get('username', '') + password = auth['passwordCredentials'].get('password', '') + tenant = auth.get('tenantName', None) + + (user_ref, tenant_ref, extras) = \ + self.identity_api.authenticate(user_id=username, + password=password, + tenant_id=tenant) + token_ref = self.token_api.create_token(user=user_ref, + tenant=tenant_ref, + extras=extras) + catalog_ref = self.catalog_api.get_catalog(user=user_ref, + tenant=tenant_ref, + extras=extras) + + elif 'tokenCredentials' in auth: + token = auth['tokenCredentials'].get('token', None) + tenant = auth.get('tenantName') + + old_token_ref = self.token_api.get_token(token_id=token) + user_ref = old_token_ref['user'] + + assert tenant in user_ref['tenants'] + + tenant_ref = self.identity_api.get_tenant(tenant) + extras = self.identity_api.get_extras( + user_id=user_ref['id'], + tenant_id=tenant_ref['tenant']['id']) + token_ref = self.token_api.create_token(user=user_ref, + tenant=tenant_ref, + extras=extras) + catalog_ref = self.catalog_api.get_catalog( + user=user_ref, + tenant=tenant_ref, + extras=extras) + + return self._format_authenticate(token_ref, catalog_ref) + + def _format_authenticate(sef, token_ref, catalog_ref): + return {} + + #admin-only + def validate_token(self, context, token_id, belongs_to=None): + """Check that a token is valid. + + Optionally, also ensure that it is owned by a specific tenant. + + """ + token_ref = self.token_api.get_token(token_id) + if belongs_to: + assert token_ref['tenant']['id'] == belongs_to + return self._format_token(token_ref) + + def _format_token(self, token_ref): + return {} -class KeystoneTokenController(service.TokenController): - def validate_token(self, context, token_id): - token = super(KeystoneTokenController, self).validate_token( - context, token_id) - # TODO(termie): munge into keystone format + def tenants_for_token(self, context): + """Get valid tenants for token based on token used to authenticate. - tenants = [{'tenantId': token['tenant']['id'], - 'name': token['tenant']['name']}] - roles = [] - if token['extras'].get('is_admin'): - roles.append({ - 'roleId': 'Admin', - 'href': 'https://www.openstack.org/identity/v2.0/roles/admin', - 'tenantId': token['tenant']['id']}) + Pulls the token from the context, validates it and gets the valid + tenants for the user in the token. - return {'auth': {'token': _token_to_keystone(token), - 'user': {'groups': {'group': tenants}, - 'roleRefs': roles, - 'username': token['user']['name'], - 'tenantId': token['tenant']['id']}}} + Doesn't care about token scopedness. + """ + token_ref = self.token_api.get_token(context['token_id']) + user_ref = token_ref['user'] + tenant_refs = [] + for tenant_id in user_ref['tenants']: + tenant_refs.append(self.identity_api.get_tenant(tenant_id)) + return self._format_tenants_for_token(tenant_refs) + + def _format_tenants_for_token(self, tenant_refs): + return [{}] + + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return KeystoneRouter(conf) diff --git a/keystonelight/service.py b/keystonelight/service.py index a4075fd077..32feaba1e2 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -49,7 +49,9 @@ class TokenAuthMiddleware(wsgi.Middleware): logging.info('GOT HEADERS %s', request.headers) token = request.headers.get('X-Auth-Token') logging.info('GOT TOKEN %s', token) - request.environ['openstack.context'] = {'token': token} + context = request.environ.get('openstack.context', {}) + context['token_id'] = token + request.environ['openstack.context'] = context class PostParamsMiddleware(wsgi.Middleware): diff --git a/keystonelight/token.py b/keystonelight/token.py index f5e262ffd8..34a1930801 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -19,9 +19,9 @@ class Manager(object): token_ref = self.driver.create_token(token, data) return token_ref - def validate_token(self, context, token_id): + def get_token(self, context, token_id): """Return info for a token if it is valid.""" return self.driver.get_token(token_id) - def revoke_token(self, context, token_id): + def delete_token(self, context, token_id): self.driver.delete_token(token_id) diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_HEAD.conf index 9154af1a16..511e369676 100644 --- a/tests/keystone_compat_HEAD.conf +++ b/tests/keystone_compat_HEAD.conf @@ -1,6 +1,5 @@ [app:main] -token_controller = keystonelight.keystone_compat.KeystoneTokenController -identity_controller = keystonelight.keystone_compat.KeystoneIdentityController +catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -paste.app_factory = keystonelight.service:identity_app_factory +paste.app_factory = keystonelight.keystone_compat:app_factory From ef9f0392a3614439ab5b3c9d4a79f9516f1880e5 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 12:19:32 +0300 Subject: [PATCH 012/334] move diagram into docs dir --- keystone_compat_flows.sdx => docs/keystone_compat_flows.sdx | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename keystone_compat_flows.sdx => docs/keystone_compat_flows.sdx (100%) diff --git a/keystone_compat_flows.sdx b/docs/keystone_compat_flows.sdx similarity index 100% rename from keystone_compat_flows.sdx rename to docs/keystone_compat_flows.sdx From 06944e8d69473080d74637972d6d293658469ecd Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 12:23:32 +0300 Subject: [PATCH 013/334] add context to calls --- keystonelight/keystone_compat.py | 41 ++++++++++++++++++++------------ keystonelight/test.py | 5 ++++ keystonelight/wsgi.py | 7 ++++-- tests/keystone_compat_HEAD.conf | 13 +++++++++- tests/test_keystone_compat.py | 7 ++++-- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 98c789528e..4c73b0788c 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -70,13 +70,16 @@ class KeystoneController(service.BaseApplication): tenant = auth.get('tenantName', None) (user_ref, tenant_ref, extras) = \ - self.identity_api.authenticate(user_id=username, + self.identity_api.authenticate(context=context, + user_id=username, password=password, tenant_id=tenant) - token_ref = self.token_api.create_token(user=user_ref, + token_ref = self.token_api.create_token(context=context, + user=user_ref, tenant=tenant_ref, extras=extras) - catalog_ref = self.catalog_api.get_catalog(user=user_ref, + catalog_ref = self.catalog_api.get_catalog(context=context, + user=user_ref, tenant=tenant_ref, extras=extras) @@ -84,22 +87,26 @@ class KeystoneController(service.BaseApplication): token = auth['tokenCredentials'].get('token', None) tenant = auth.get('tenantName') - old_token_ref = self.token_api.get_token(token_id=token) + old_token_ref = self.token_api.get_token(context=context, + token_id=token) user_ref = old_token_ref['user'] assert tenant in user_ref['tenants'] - tenant_ref = self.identity_api.get_tenant(tenant) + tenant_ref = self.identity_api.get_tenant(context=context, + tenant_id=tenant) extras = self.identity_api.get_extras( + context=context, user_id=user_ref['id'], tenant_id=tenant_ref['tenant']['id']) - token_ref = self.token_api.create_token(user=user_ref, - tenant=tenant_ref, - extras=extras) - catalog_ref = self.catalog_api.get_catalog( - user=user_ref, - tenant=tenant_ref, - extras=extras) + token_ref = self.token_api.create_token(context=context, + user=user_ref, + tenant=tenant_ref, + extras=extras) + catalog_ref = self.catalog_api.get_catalog(context=context, + user=user_ref, + tenant=tenant_ref, + extras=extras) return self._format_authenticate(token_ref, catalog_ref) @@ -113,7 +120,8 @@ class KeystoneController(service.BaseApplication): Optionally, also ensure that it is owned by a specific tenant. """ - token_ref = self.token_api.get_token(token_id) + token_ref = self.token_api.get_token(context=context, + token_id=token_id) if belongs_to: assert token_ref['tenant']['id'] == belongs_to return self._format_token(token_ref) @@ -131,11 +139,14 @@ class KeystoneController(service.BaseApplication): Doesn't care about token scopedness. """ - token_ref = self.token_api.get_token(context['token_id']) + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) user_ref = token_ref['user'] tenant_refs = [] for tenant_id in user_ref['tenants']: - tenant_refs.append(self.identity_api.get_tenant(tenant_id)) + tenant_refs.append(self.identity_api.get_tenant( + context=context, + tenant_id=tenant_id)) return self._format_tenants_for_token(tenant_refs) def _format_tenants_for_token(self, tenant_refs): diff --git a/keystonelight/test.py b/keystonelight/test.py index cb56f2c324..edf88bac16 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -43,6 +43,11 @@ class TestCase(unittest.TestCase): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) return deploy.loadapp(config) + def appconfig(self, config): + if not config.startswith('config:'): + config = 'config:%s.conf' % os.path.join(TESTSDIR, config) + return deploy.appconfig(config) + def client(self, app, *args, **kw): return TestClient(app, *args, **kw) diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 5eea49085a..0e2af3de28 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -182,11 +182,14 @@ class Middleware(Application): """ def _factory(app): - return cls(app, **local_config) + conf = global_config.copy() + conf.update(local_config) + return cls(app, conf) return _factory - def __init__(self, application): + def __init__(self, application, options): self.application = application + self.options = options def process_request(self, req): """Called on each request. diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_HEAD.conf index 511e369676..7f6e1f176b 100644 --- a/tests/keystone_compat_HEAD.conf +++ b/tests/keystone_compat_HEAD.conf @@ -1,5 +1,16 @@ -[app:main] +[DEFAULT] catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken + +[filter:token_auth] +paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory + +[filter:json_body] +paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory + +[app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory + +[pipeline:main] +pipeline = token_auth json_body keystone diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 95822d8f58..4a2bc1eb4c 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -74,11 +74,14 @@ class HeadCompatTestCase(CompatTestCase): revdir = checkout_samples('HEAD') self.sampledir = os.path.join(revdir, SAMPLE_DIR) self.app = self.loadapp('keystone_compat_HEAD') + self.options = self.appconfig('keystone_compat_HEAD') self.identity_backend = utils.import_object( - self.app.options['identity_driver'], options=self.app.options) + self.options['identity_driver'], options=self.options) self.token_backend = utils.import_object( - self.app.options['token_driver'], options=self.app.options) + self.options['token_driver'], options=self.options) + self.catalog_backend = utils.import_object( + self.options['catalog_driver'], options=self.options) super(HeadCompatTestCase, self).setUp() From f886ab990d169097ac815ca83c78f995f2625936 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 12:42:03 +0300 Subject: [PATCH 014/334] flow working, added debugging --- keystonelight/keystone_compat.py | 1 - keystonelight/service.py | 7 ++----- keystonelight/test.py | 4 ++++ keystonelight/wsgi.py | 13 +++++++------ tests/keystone_compat_HEAD.conf | 5 ++++- tests/test_keystone_compat.py | 8 ++++---- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 4c73b0788c..d47bbcd5ca 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -129,7 +129,6 @@ class KeystoneController(service.BaseApplication): def _format_token(self, token_ref): return {} - def tenants_for_token(self, context): """Get valid tenants for token based on token used to authenticate. diff --git a/keystonelight/service.py b/keystonelight/service.py index 32feaba1e2..c607f8aa7c 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -22,7 +22,7 @@ class BaseApplication(wsgi.Application): action = arg_dict['action'] del arg_dict['action'] del arg_dict['controller'] - logging.info('arg_dict: %s', arg_dict) + logging.debug('arg_dict: %s', arg_dict) context = req.environ.get('openstack.context', {}) # allow middleware up the stack to override the params @@ -46,9 +46,7 @@ class BaseApplication(wsgi.Application): class TokenAuthMiddleware(wsgi.Middleware): def process_request(self, request): - logging.info('GOT HEADERS %s', request.headers) token = request.headers.get('X-Auth-Token') - logging.info('GOT TOKEN %s', token) context = request.environ.get('openstack.context', {}) context['token_id'] = token request.environ['openstack.context'] = context @@ -142,12 +140,11 @@ class IdentityController(BaseApplication): dict(tenant=tenant, user=user, extras=extras)) - logging.info(token) + logging.debug('TOKEN: %s', token) return token def get_tenants(self, context): token_id = context.get('token') - logging.info("GET TENANTS %s", token_id) token = self.token_api.validate_token(context, token_id) return self.identity_api.get_tenants(context, diff --git a/keystonelight/test.py b/keystonelight/test.py index edf88bac16..98557c7fc1 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -19,6 +19,10 @@ class TestClient(object): def request(self, method, path, headers=None, body=None): if headers is None: headers = {} + + if self.token: + headers.setdefault('X-Auth-Token', self.token) + req = wsgi.Request.blank(path) req.method = method for k, v in headers.iteritems(): diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 0e2af3de28..b1aedad60a 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -224,16 +224,17 @@ class Debug(Middleware): @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): - print ('*' * 40) + ' REQUEST ENVIRON' + logging.debug('%s %s %s', ('*' * 20), 'REQUEST ENVIRON', ('*' * 20)) for key, value in req.environ.items(): - print key, '=', value - print + logging.debug('%s = %s', key, value) + logging.debug('') + resp = req.get_response(self.application) - print ('*' * 40) + ' RESPONSE HEADERS' + logging.debug('%s %s %s', ('*' * 20), 'RESPONSE HEADERS', ('*' * 20)) for (key, value) in resp.headers.iteritems(): - print key, '=', value - print + logging.debug('%s = %s', key, value) + logging.debug('') resp.app_iter = self.print_generator(resp.app_iter) diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_HEAD.conf index 7f6e1f176b..d2fa70c589 100644 --- a/tests/keystone_compat_HEAD.conf +++ b/tests/keystone_compat_HEAD.conf @@ -3,6 +3,9 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +[filter:debug] +paste.filter_factory = keystonelight.wsgi:Debug.factory + [filter:token_auth] paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory @@ -13,4 +16,4 @@ paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory paste.app_factory = keystonelight.keystone_compat:app_factory [pipeline:main] -pipeline = token_auth json_body keystone +pipeline = token_auth json_body debug keystone diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 4a2bc1eb4c..0d8cc01451 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -61,12 +61,12 @@ class CompatTestCase(test.TestCase): self.token_foo_unscoped = self.token_backend.create_token( 'foo_unscoped', models.Token(id='foo_unscoped', - user='foo')) + user=self.user_foo)) self.token_foo_scoped = self.token_backend.create_token( 'foo_scoped', - models.Token(id='foo_unscoped', - user='foo', - tenant='1234')) + models.Token(id='foo_scoped', + user=self.user_foo, + tenant=self.tenant_1234)) class HeadCompatTestCase(CompatTestCase): From a98b2ed7064d01696b8fc980cf90cefd7cbbd6d7 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 13:28:22 +0300 Subject: [PATCH 015/334] get tenants passing, yay --- keystonelight/backends/kvs.py | 8 +++++++ keystonelight/identity.py | 7 ++++-- keystonelight/keystone_compat.py | 3 ++- keystonelight/logging.py | 19 +++++++++++++++ keystonelight/models.py | 2 +- keystonelight/test.py | 41 +++++++++++++++++++++++++++++--- keystonelight/token.py | 2 ++ tests/test_keystone_compat.py | 8 +++---- 8 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 keystonelight/logging.py diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 807492e05b..246d2db80e 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -23,6 +23,14 @@ class KvsIdentity(object): return o + def get_tenant(self, tenant_id): + tenant_ref = self.db.get('tenant-%s' % tenant_id) + return tenant_ref + + def get_user(self, user_id): + user_ref = self.db.get('user-%s' % user_id) + return user_ref + # Private CRUD for testing def _create_user(self, id, user): self.db.set('user-%s' % id, user) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 9b887038da..9270719143 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -20,5 +20,8 @@ class Manager(object): """ return self.driver.authenticate(**kwargs) - def get_tenants(self, context, user_id): - return self.driver.get_tenants(user_id) + def get_user(self, context, user_id): + return self.driver.get_user(user_id) + + def get_tenant(self, context, tenant_id): + return self.driver.get_tenant(tenant_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index d47bbcd5ca..e776f7a1d0 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -149,7 +149,8 @@ class KeystoneController(service.BaseApplication): return self._format_tenants_for_token(tenant_refs) def _format_tenants_for_token(self, tenant_refs): - return [{}] + o = {'tenants': {'values': tenant_refs}} + return o def app_factory(global_conf, **local_conf): diff --git a/keystonelight/logging.py b/keystonelight/logging.py new file mode 100644 index 0000000000..db4df55125 --- /dev/null +++ b/keystonelight/logging.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import functools +import logging +import pprint + + +from logging import * + + +def log_debug(f): + @functools.wraps(f) + def wrapper(*args, **kw): + logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw)) + rv = f(*args, **kw) + logging.debug(pprint.pformat(rv, indent=2)) + logging.debug('') + return rv + return wrapper diff --git a/keystonelight/models.py b/keystonelight/models.py index a56e8061ad..08747cddb5 100644 --- a/keystonelight/models.py +++ b/keystonelight/models.py @@ -9,7 +9,7 @@ class User(dict): def __init__(self, id=None, tenants=None, *args, **kw): if tenants is None: tenants = [] - super(User, self).__init__(id=id, tenants=[], *args, **kw) + super(User, self).__init__(id=id, tenants=tenants, *args, **kw) class Tenant(dict): diff --git a/keystonelight/test.py b/keystonelight/test.py index 98557c7fc1..d5409de42e 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -55,14 +55,49 @@ class TestCase(unittest.TestCase): def client(self, app, *args, **kw): return TestClient(app, *args, **kw) + + def assertListEquals(self, expected, actual): + copy = expected[:] + self.assertEquals(len(expected), len(actual)) + while copy: + item = copy.pop() + matched = False + for x in actual: + #print 'COMPARE', item, x, + try: + self.assertDeepEquals(item, x) + matched = True + #print 'MATCHED' + break + except AssertionError as e: + #print e + pass + if not matched: + raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) + + def assertDictEquals(self, expected, actual): for k in expected: self.assertTrue(k in actual, "Expected key %s not in %s." % (k, actual)) - self.assertEquals(expected[k], actual[k], - "Expected value for %s to be '%s', not '%s'." - % (k, expected[k], actual[k])) + self.assertDeepEquals(expected[k], actual[k]) + for k in actual: self.assertTrue(k in expected, "Unexpected key %s in %s." % (k, actual)) + + + def assertDeepEquals(self, expected, actual): + try: + if type(expected) is type([]) or type(expected) is type(tuple()): + # assert items equal, ignore order + self.assertListEquals(expected, actual) + elif type(expected) is type({}): + self.assertDictEquals(expected, actual) + else: + self.assertEquals(expected, actual) + except AssertionError as e: + raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) + + diff --git a/keystonelight/token.py b/keystonelight/token.py index 34a1930801..a4c814e572 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -4,6 +4,7 @@ import uuid +from keystonelight import logging from keystonelight import utils @@ -19,6 +20,7 @@ class Manager(object): token_ref = self.driver.create_token(token, data) return token_ref + @logging.log_debug def get_token(self, context, token_id): """Return info for a token if it is valid.""" return self.driver.get_token(token_id) diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 0d8cc01451..9a75dcd70b 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -49,13 +49,13 @@ class CompatTestCase(test.TestCase): '1234', models.Tenant(id='1234', name='ACME Corp', - description='A description...', + description='A description ...', enabled=True)) self.tenant_3456 = self.identity_backend._create_tenant( '3456', models.Tenant(id='3456', name='Iron Works', - description='A description...', + description='A description ...', enabled=True)) self.token_foo_unscoped = self.token_backend.create_token( @@ -90,11 +90,11 @@ class HeadCompatTestCase(CompatTestCase): client = self.client(self.app, token=self.token_foo_unscoped['id']) resp = client.get('/v2.0/tenants') data = json.loads(resp.body) - self.assertDictEquals(self.tenants_for_token, data) + self.assertDeepEquals(self.tenants_for_token, data) def test_tenants_for_token_scoped(self): # get_tenants_for_token client = self.client(self.app, token=self.token_foo_scoped['id']) resp = client.get('/v2.0/tenants') data = json.loads(resp.body) - self.assertDictEquals(self.tenants_for_token, data) + self.assertDeepEquals(self.tenants_for_token, data) From ba4913f4630997600c9256bc5a269431d4934dbb Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 15:08:09 +0300 Subject: [PATCH 016/334] base tests on keystone-diablo/stable --- keystonelight/backends/kvs.py | 8 --- keystonelight/keystone_compat.py | 18 ++++++- keystonelight/test.py | 12 ++--- ..._HEAD.conf => keystone_compat_diablo.conf} | 0 tests/test_keystone_compat.py | 52 +++++++++++++++---- 5 files changed, 62 insertions(+), 28 deletions(-) rename tests/{keystone_compat_HEAD.conf => keystone_compat_diablo.conf} (100%) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 246d2db80e..5616ee7d80 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -15,14 +15,6 @@ class KvsIdentity(object): self.db = db # Public interface - def tenants_for_user(self, user_id): - user = self.db.get('user-%s' % user_id) - o = [] - for tenant_id in user['tenants']: - o.append(self.db.get('tenant-%s' % tenant_id)) - - return o - def get_tenant(self, tenant_id): tenant_ref = self.db.get('tenant-%s' % tenant_id) return tenant_ref diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index e776f7a1d0..af1ff0b12c 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -127,7 +127,20 @@ class KeystoneController(service.BaseApplication): return self._format_token(token_ref) def _format_token(self, token_ref): - return {} + user_ref = token_ref['user'] + o = {'access': {'token': {'id': token_ref['id'], + 'expires': token_ref['expires'] + }, + 'user': {'id': user_ref['id'], + 'name': user_ref['name'], + 'roles': user_ref['roles'] or [], + 'roles_links': user_ref['roles_links'] or [] + } + } + } + if 'tenant' in token_ref: + o['access']['token']['tenant'] = token_ref['tenant'] + return o def tenants_for_token(self, context): """Get valid tenants for token based on token used to authenticate. @@ -149,7 +162,8 @@ class KeystoneController(service.BaseApplication): return self._format_tenants_for_token(tenant_refs) def _format_tenants_for_token(self, tenant_refs): - o = {'tenants': {'values': tenant_refs}} + o = {'tenants': tenant_refs, + 'tenants_links': []} return o diff --git a/keystonelight/test.py b/keystonelight/test.py index d5409de42e..c320a18c9f 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -55,27 +55,26 @@ class TestCase(unittest.TestCase): def client(self, app, *args, **kw): return TestClient(app, *args, **kw) - def assertListEquals(self, expected, actual): copy = expected[:] + print expected, actual self.assertEquals(len(expected), len(actual)) while copy: item = copy.pop() matched = False for x in actual: - #print 'COMPARE', item, x, + print 'COMPARE', item, x, try: self.assertDeepEquals(item, x) matched = True - #print 'MATCHED' + print 'MATCHED' break except AssertionError as e: - #print e + print e pass if not matched: raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) - def assertDictEquals(self, expected, actual): for k in expected: self.assertTrue(k in actual, @@ -86,8 +85,6 @@ class TestCase(unittest.TestCase): self.assertTrue(k in expected, "Unexpected key %s in %s." % (k, actual)) - - def assertDeepEquals(self, expected, actual): try: if type(expected) is type([]) or type(expected) is type(tuple()): @@ -98,6 +95,7 @@ class TestCase(unittest.TestCase): else: self.assertEquals(expected, actual) except AssertionError as e: + raise raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) diff --git a/tests/keystone_compat_HEAD.conf b/tests/keystone_compat_diablo.conf similarity index 100% rename from tests/keystone_compat_HEAD.conf rename to tests/keystone_compat_diablo.conf diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 9a75dcd70b..deda16a0fe 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -8,9 +8,11 @@ from keystonelight import utils IDENTITY_API_REPO = 'git://github.com/openstack/identity-api.git' +KEYSTONE_REPO = 'git://github.com/openstack/keystone.git' -SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' +IDENTITY_SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' +KEYSTONE_SAMPLE_DIR = 'keystone/content/common/samples' cd = os.chdir @@ -18,14 +20,14 @@ cd = os.chdir def checkout_samples(rev): """Make sure we have a checkout of the API docs.""" - revdir = os.path.join(test.VENDOR, 'identity-api-%s' % rev) + revdir = os.path.join(test.VENDOR, 'keystone-%s' % rev.replace('/', '_')) if not os.path.exists(revdir): - utils.git('clone', IDENTITY_API_REPO, revdir) + utils.git('clone', KEYSTONE_REPO, revdir) cd(revdir) utils.git('pull') - utils.git('checkout', rev) + utils.git('checkout', '-q', rev) return revdir @@ -40,8 +42,30 @@ class CompatTestCase(test.TestCase): self.tenants_for_token = json.load(open( os.path.join(self.sampledir, 'tenants.json'))) + self.validate_token = json.load(open( + os.path.join(self.sampledir, 'validatetoken.json'))) - # For the tenants for token call + # validate_token call + self.tenant_345 = self.identity_backend._create_tenant( + '345', + models.Tenant(id='345', name='My Project')) + self.user_123 = self.identity_backend._create_user( + '123', + models.User(id='123', name='jqsmith', tenants=[self.tenant_345['id']], + roles=[{'id': '234', + 'name': 'compute:admin'}, + {'id': '234', + 'name': 'object-store:admin', + 'tenantId': '1'}], + roles_links=[])) + self.token_123 = self.token_backend.create_token( + 'ab48a9efdfedb23ty3494', + models.Token(id='ab48a9efdfedb23ty3494', + expires='2010-11-01T03:32:15-05:00', + user=self.user_123, + tenant=self.tenant_345)) + + # tenants_for_token call self.user_foo = self.identity_backend._create_user( 'foo', models.User(id='foo', tenants=['1234', '3456'])) @@ -69,12 +93,12 @@ class CompatTestCase(test.TestCase): tenant=self.tenant_1234)) -class HeadCompatTestCase(CompatTestCase): +class DiabloCompatTestCase(CompatTestCase): def setUp(self): - revdir = checkout_samples('HEAD') - self.sampledir = os.path.join(revdir, SAMPLE_DIR) - self.app = self.loadapp('keystone_compat_HEAD') - self.options = self.appconfig('keystone_compat_HEAD') + revdir = checkout_samples('stable/diablo') + self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) + self.app = self.loadapp('keystone_compat_diablo') + self.options = self.appconfig('keystone_compat_diablo') self.identity_backend = utils.import_object( self.options['identity_driver'], options=self.options) @@ -83,7 +107,13 @@ class HeadCompatTestCase(CompatTestCase): self.catalog_backend = utils.import_object( self.options['catalog_driver'], options=self.options) - super(HeadCompatTestCase, self).setUp() + super(DiabloCompatTestCase, self).setUp() + + def test_validate_token_scoped(self): + client = self.client(self.app, token=self.token_123['id']) + resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) + data = json.loads(resp.body) + self.assertDeepEquals(self.validate_token, data) def test_tenants_for_token_unscoped(self): # get_tenants_for_token From d920d8432aa465c2ef19d66af1c3f8726985a9d2 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 19 Oct 2011 16:23:33 +0300 Subject: [PATCH 017/334] authenticate working, too --- keystonelight/backends/kvs.py | 29 +++++++- keystonelight/catalog.py | 4 +- keystonelight/keystone_compat.py | 51 ++++++++------ keystonelight/service.py | 1 + keystonelight/test.py | 10 +-- keystonelight/wsgi.py | 9 ++- ...keystone_compat_diablo_sample_catalog.json | 53 +++++++++++++++ tests/test_keystone_compat.py | 68 +++++++++++++++---- 8 files changed, 179 insertions(+), 46 deletions(-) create mode 100644 tests/keystone_compat_diablo_sample_catalog.json diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 5616ee7d80..175637fe5f 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -15,6 +15,18 @@ class KvsIdentity(object): self.db = db # Public interface + def authenticate(self, user_id=None, tenant_id=None, password=None): + user_ref = self.get_user(user_id) + tenant_ref = None + extras_ref = None + if user_ref['password'] != password: + raise AssertionError('Invalid user / password') + + if tenant_id and tenant_id in user_ref['tenants']: + tenant_ref = self.get_tenant(tenant_id) + extras_ref = self.get_extras(user_id, tenant_id) + return (user_ref, tenant_ref, extras_ref) + def get_tenant(self, tenant_id): tenant_ref = self.db.get('tenant-%s' % tenant_id) return tenant_ref @@ -23,6 +35,9 @@ class KvsIdentity(object): user_ref = self.db.get('user-%s' % user_id) return user_ref + def get_extras(self, user_id, tenant_id): + return self.db.get('extras-%s-%s' % (user_id, tenant_id)) + # Private CRUD for testing def _create_user(self, id, user): self.db.set('user-%s' % id, user) @@ -32,6 +47,12 @@ class KvsIdentity(object): self.db.set('tenant-%s' % id, tenant) return tenant + def _create_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (user_id, tenant_id), extras) + return extras + + + class KvsToken(object): def __init__(self, options, db=None): @@ -58,5 +79,9 @@ class KvsCatalog(object): self.db = db # Public interface - def get_catalog(self, user, tenant, extras=None): - return self.db.get('catalog-%s' % tenant['id']) + def get_catalog(self, user_id, tenant_id, extras=None): + return self.db.get('catalog-%s' % tenant_id) + + # Private interface + def _create_catalog(self, user_id, tenant_id, data): + self.db.set('catalog-%s' % tenant_id, data) diff --git a/keystonelight/catalog.py b/keystonelight/catalog.py index 99b870205d..221e1e4873 100644 --- a/keystonelight/catalog.py +++ b/keystonelight/catalog.py @@ -13,6 +13,6 @@ class Manager(object): self.driver = utils.import_object(options['catalog_driver'], options=options) - def get_catalog(self, user, tenant, extras=None): + def get_catalog(self, context, user_id, tenant_id, extras=None): """Return info for a catalog if it is valid.""" - return self.driver.get_catalog(user, tenant, extras=extras) + return self.driver.get_catalog(user_id, tenant_id, extras=extras) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index af1ff0b12c..7a9d16e734 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -69,19 +69,21 @@ class KeystoneController(service.BaseApplication): password = auth['passwordCredentials'].get('password', '') tenant = auth.get('tenantName', None) - (user_ref, tenant_ref, extras) = \ + (user_ref, tenant_ref, extras_ref) = \ self.identity_api.authenticate(context=context, user_id=username, password=password, tenant_id=tenant) - token_ref = self.token_api.create_token(context=context, - user=user_ref, - tenant=tenant_ref, - extras=extras) - catalog_ref = self.catalog_api.get_catalog(context=context, - user=user_ref, - tenant=tenant_ref, - extras=extras) + token_ref = self.token_api.create_token(context, + dict(expires='', + user=user_ref, + tenant=tenant_ref, + extras=extras_ref)) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + extras=extras_ref) elif 'tokenCredentials' in auth: token = auth['tokenCredentials'].get('token', None) @@ -95,23 +97,27 @@ class KeystoneController(service.BaseApplication): tenant_ref = self.identity_api.get_tenant(context=context, tenant_id=tenant) - extras = self.identity_api.get_extras( + extras_ref = self.identity_api.get_extras( context=context, user_id=user_ref['id'], tenant_id=tenant_ref['tenant']['id']) - token_ref = self.token_api.create_token(context=context, - user=user_ref, - tenant=tenant_ref, - extras=extras) - catalog_ref = self.catalog_api.get_catalog(context=context, - user=user_ref, - tenant=tenant_ref, - extras=extras) + token_ref = self.token_api.create_token(context, + dict(expires='', + user=user_ref, + tenant=tenant_ref, + extras=extras_ref)) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + extras=extras_ref) return self._format_authenticate(token_ref, catalog_ref) - def _format_authenticate(sef, token_ref, catalog_ref): - return {} + def _format_authenticate(self, token_ref, catalog_ref): + o = self._format_token(token_ref) + o['access']['serviceCatalog'] = catalog_ref + return o #admin-only def validate_token(self, context, token_id, belongs_to=None): @@ -128,13 +134,14 @@ class KeystoneController(service.BaseApplication): def _format_token(self, token_ref): user_ref = token_ref['user'] + extras_ref = token_ref['extras'] o = {'access': {'token': {'id': token_ref['id'], 'expires': token_ref['expires'] }, 'user': {'id': user_ref['id'], 'name': user_ref['name'], - 'roles': user_ref['roles'] or [], - 'roles_links': user_ref['roles_links'] or [] + 'roles': extras_ref['roles'] or [], + 'roles_links': extras_ref['roles_links'] or [] } } } diff --git a/keystonelight/service.py b/keystonelight/service.py index c607f8aa7c..78b6b8ecda 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -94,6 +94,7 @@ class JsonBodyMiddleware(wsgi.Middleware): params_json = request.body if not params_json: + print "ASDASDASDASD" return params_parsed = json.loads(params_json) diff --git a/keystonelight/test.py b/keystonelight/test.py index c320a18c9f..35a0f9b045 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -27,7 +27,7 @@ class TestClient(object): req.method = method for k, v in headers.iteritems(): req.headers[k] = v - if req.body: + if body: req.body = body return req.get_response(self.app) @@ -57,20 +57,20 @@ class TestCase(unittest.TestCase): def assertListEquals(self, expected, actual): copy = expected[:] - print expected, actual + #print expected, actual self.assertEquals(len(expected), len(actual)) while copy: item = copy.pop() matched = False for x in actual: - print 'COMPARE', item, x, + #print 'COMPARE', item, x, try: self.assertDeepEquals(item, x) matched = True - print 'MATCHED' + #print 'MATCHED' break except AssertionError as e: - print e + #print e pass if not matched: raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index b1aedad60a..5be3170e98 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -228,6 +228,11 @@ class Debug(Middleware): for key, value in req.environ.items(): logging.debug('%s = %s', key, value) logging.debug('') + logging.debug('%s %s %s', ('*' * 20), 'REQUEST BODY', ('*' * 20)) + for line in req.body_file: + logging.debug(line) + logging.debug('') + resp = req.get_response(self.application) @@ -243,10 +248,10 @@ class Debug(Middleware): @staticmethod def print_generator(app_iter): """Iterator that prints the contents of a wrapper string.""" - print ('*' * 40) + ' BODY' + logging.debug('%s %s %s', ('*' * 20), 'RESPONSE BODY', ('*' * 20)) for part in app_iter: sys.stdout.write(part) - sys.stdout.flush() + #sys.stdout.flush() yield part print diff --git a/tests/keystone_compat_diablo_sample_catalog.json b/tests/keystone_compat_diablo_sample_catalog.json new file mode 100644 index 0000000000..1ccef4a1ed --- /dev/null +++ b/tests/keystone_compat_diablo_sample_catalog.json @@ -0,0 +1,53 @@ +[{ + "name":"Cloud Servers", + "type":"compute", + "endpoints":[{ + "tenantId":"1", + "publicURL":"https://compute.north.host/v1/1234", + "internalURL":"https://compute.north.host/v1/1234", + "region":"North", + "versionId":"1.0", + "versionInfo":"https://compute.north.host/v1.0/", + "versionList":"https://compute.north.host/" + }, + { + "tenantId":"2", + "publicURL":"https://compute.north.host/v1.1/3456", + "internalURL":"https://compute.north.host/v1.1/3456", + "region":"North", + "versionId":"1.1", + "versionInfo":"https://compute.north.host/v1.1/", + "versionList":"https://compute.north.host/" + } + ], + "endpoints_links":[] +}, +{ + "name":"Cloud Files", + "type":"object-store", + "endpoints":[{ + "tenantId":"11", + "publicURL":"https://compute.north.host/v1/blah-blah", + "internalURL":"https://compute.north.host/v1/blah-blah", + "region":"South", + "versionId":"1.0", + "versionInfo":"uri", + "versionList":"uri" + }, + { + "tenantId":"2", + "publicURL":"https://compute.north.host/v1.1/blah-blah", + "internalURL":"https://compute.north.host/v1.1/blah-blah", + "region":"South", + "versionId":"1.1", + "versionInfo":"https://compute.north.host/v1.1/", + "versionList":"https://compute.north.host/" + } + ], + "endpoints_links":[{ + "rel":"next", + "href":"https://identity.north.host/v2.0/endpoints?marker=2" + } + ] +} +] diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index deda16a0fe..83035ff134 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -2,6 +2,7 @@ import copy import os import json +from keystonelight import logging from keystonelight import models from keystonelight import test from keystonelight import utils @@ -35,15 +36,16 @@ class CompatTestCase(test.TestCase): def setUp(self): super(CompatTestCase, self).setUp() - self.auth_creds = json.load(open( - os.path.join(self.sampledir, 'auth_credentials.json'))) - self.auth_creds_notenant = copy.deepcopy(self.auth_creds) - self.auth_creds_notenant['auth'].pop('tenantName', None) - self.tenants_for_token = json.load(open( os.path.join(self.sampledir, 'tenants.json'))) self.validate_token = json.load(open( os.path.join(self.sampledir, 'validatetoken.json'))) + # NOTE(termie): stupid hack to deal with the keystone samples being + # completely inconsistent + self.validate_token['access']['user']['roles'][1]['id'] = u'235' + + self.auth_response = json.load(open( + os.path.join(self.sampledir, 'auth.json'))) # validate_token call self.tenant_345 = self.identity_backend._create_tenant( @@ -51,19 +53,36 @@ class CompatTestCase(test.TestCase): models.Tenant(id='345', name='My Project')) self.user_123 = self.identity_backend._create_user( '123', - models.User(id='123', name='jqsmith', tenants=[self.tenant_345['id']], - roles=[{'id': '234', - 'name': 'compute:admin'}, - {'id': '234', - 'name': 'object-store:admin', - 'tenantId': '1'}], - roles_links=[])) + models.User(id='123', + name='jqsmith', + tenants=[self.tenant_345['id']], + password='password')) + self.extras_123 = self.identity_backend._create_extras( + self.user_123['id'], self.tenant_345['id'], + dict(roles=[{'id': '234', + 'name': 'compute:admin'}, + {'id': '235', + 'name': 'object-store:admin', + 'tenantId': '1'}], + roles_links=[])) self.token_123 = self.token_backend.create_token( 'ab48a9efdfedb23ty3494', models.Token(id='ab48a9efdfedb23ty3494', expires='2010-11-01T03:32:15-05:00', user=self.user_123, - tenant=self.tenant_345)) + tenant=self.tenant_345, + extras=self.extras_123)) + + # auth call + # NOTE(termie): the service catalog in the sample doesn't really have + # anything to do with the auth being returned, so just load + # it fully from a fixture and add it to our db + catalog = json.load(open( + os.path.join(os.path.dirname(__file__), + 'keystone_compat_diablo_sample_catalog.json'))) + self.catalog_backend._create_catalog(self.user_123['id'], + self.tenant_345['id'], + catalog) # tenants_for_token call self.user_foo = self.identity_backend._create_user( @@ -109,6 +128,29 @@ class DiabloCompatTestCase(CompatTestCase): super(DiabloCompatTestCase, self).setUp() + def test_authenticate_scoped(self): + client = self.client(self.app) + post_data = json.dumps( + {'auth': {'passwordCredentials': {'username': self.user_123['id'], + 'password': self.user_123['password'], + }, + 'tenantName': self.tenant_345['id']}}) + + resp = client.post('/v2.0/tokens', body=post_data) + data = json.loads(resp.body) + logging.debug('KEYS: %s', data['access'].keys()) + self.assert_('expires' in data['access']['token']) + self.assertDeepEquals(self.auth_response['access']['user'], + data['access']['user']) + self.assertDeepEquals(self.auth_response['access']['serviceCatalog'], + data['access']['serviceCatalog']) + + def test_validate_token_scoped(self): + client = self.client(self.app, token=self.token_123['id']) + resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) + data = json.loads(resp.body) + self.assertDeepEquals(self.validate_token, data) + def test_validate_token_scoped(self): client = self.client(self.app, token=self.token_123['id']) resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) From 583e3c93e134ce05b32bedaf039e9da7ab2d3146 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 26 Oct 2011 17:01:11 -0700 Subject: [PATCH 018/334] get a checkout of keystoneclient --- keystonelight/backends/kvs.py | 7 +++-- keystonelight/logging.py | 30 ++++++++++++++++++++- keystonelight/test.py | 35 ++++++++++++++++++++++++ keystonelight/utils.py | 3 ++- tests/test_keystone_compat.py | 22 +++------------- tests/test_keystoneclient_compat.py | 41 +++++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 tests/test_keystoneclient_compat.py diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 175637fe5f..6dfa2c455f 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -36,7 +36,7 @@ class KvsIdentity(object): return user_ref def get_extras(self, user_id, tenant_id): - return self.db.get('extras-%s-%s' % (user_id, tenant_id)) + return self.db.get('extras-%s-%s' % (tenant_id, user_id)) # Private CRUD for testing def _create_user(self, id, user): @@ -45,15 +45,14 @@ class KvsIdentity(object): def _create_tenant(self, id, tenant): self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) return tenant def _create_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (user_id, tenant_id), extras) + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) return extras - - class KvsToken(object): def __init__(self, options, db=None): if db is None: diff --git a/keystonelight/logging.py b/keystonelight/logging.py index db4df55125..1de4bdadc2 100644 --- a/keystonelight/logging.py +++ b/keystonelight/logging.py @@ -5,7 +5,35 @@ import logging import pprint -from logging import * +# A list of things we want to replicate from logging. +# levels +CRITICAL = logging.CRITICAL +FATAL = logging.FATAL +ERROR = logging.ERROR +WARNING = logging.WARNING +WARN = logging.WARN +INFO = logging.INFO +DEBUG = logging.DEBUG +NOTSET = logging.NOTSET + + +# methods +getLogger = logging.getLogger +debug = logging.debug +info = logging.info +warning = logging.warning +warn = logging.warn +error = logging.error +exception = logging.exception +critical = logging.critical +log = logging.log + + +# handlers +StreamHandler = logging.StreamHandler +WatchedFileHandler = logging.handlers.WatchedFileHandler +# logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. +SysLogHandler = logging.handlers.SysLogHandler def log_debug(f): diff --git a/keystonelight/test.py b/keystonelight/test.py index 35a0f9b045..0c22f3777b 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -1,8 +1,10 @@ import os import unittest +import sys from paste import deploy +from keystonelight import utils from keystonelight import wsgi @@ -11,6 +13,25 @@ VENDOR = os.path.join(ROOTDIR, 'vendor') TESTSDIR = os.path.join(ROOTDIR, 'tests') +cd = os.chdir + + +def checkout_vendor(repo, rev): + name = repo.split('/')[-1] + if name.endswith('.git'): + name = name[:-4] + + revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) + + if not os.path.exists(revdir): + utils.git('clone', repo, revdir) + + cd(revdir) + utils.git('pull') + utils.git('checkout', '-q', rev) + return revdir + + class TestClient(object): def __init__(self, app=None, token=None): self.app = app @@ -42,6 +63,16 @@ class TestClient(object): class TestCase(unittest.TestCase): + def __init__(self, *args, **kw): + super(TestCase, self).__init__(*args, **kw) + self._paths = [] + + def tearDown(self): + for path in self._paths: + if path in sys.path: + sys.path.remove(path) + super(TestCase, self).tearDown() + def loadapp(self, config): if not config.startswith('config:'): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) @@ -55,6 +86,10 @@ class TestCase(unittest.TestCase): def client(self, app, *args, **kw): return TestClient(app, *args, **kw) + def add_path(self, path): + sys.path.insert(0, path) + self._paths.append(path) + def assertListEquals(self, expected, actual): copy = expected[:] #print expected, actual diff --git a/keystonelight/utils.py b/keystonelight/utils.py index fcc37f9a43..1c006e1cc5 100644 --- a/keystonelight/utils.py +++ b/keystonelight/utils.py @@ -17,10 +17,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import subprocess import sys +from keystonelight import logging + def import_class(import_str): """Returns a class from a string including module and class.""" diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 83035ff134..2d956f1d36 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -1,6 +1,7 @@ import copy -import os import json +import os +import sys from keystonelight import logging from keystonelight import models @@ -10,28 +11,13 @@ from keystonelight import utils IDENTITY_API_REPO = 'git://github.com/openstack/identity-api.git' KEYSTONE_REPO = 'git://github.com/openstack/keystone.git' +NOVACLIENT_REPO = 'git://github.com/rackspace/python-novaclient.git' IDENTITY_SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' KEYSTONE_SAMPLE_DIR = 'keystone/content/common/samples' -cd = os.chdir - - -def checkout_samples(rev): - """Make sure we have a checkout of the API docs.""" - revdir = os.path.join(test.VENDOR, 'keystone-%s' % rev.replace('/', '_')) - - if not os.path.exists(revdir): - utils.git('clone', KEYSTONE_REPO, revdir) - - cd(revdir) - utils.git('pull') - utils.git('checkout', '-q', rev) - return revdir - - class CompatTestCase(test.TestCase): def setUp(self): super(CompatTestCase, self).setUp() @@ -114,7 +100,7 @@ class CompatTestCase(test.TestCase): class DiabloCompatTestCase(CompatTestCase): def setUp(self): - revdir = checkout_samples('stable/diablo') + revdir = test.checkout_vendor(KEYSTONE_REPO, 'stable/diablo') self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) self.app = self.loadapp('keystone_compat_diablo') self.options = self.appconfig('keystone_compat_diablo') diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py new file mode 100644 index 0000000000..b94d60448a --- /dev/null +++ b/tests/test_keystoneclient_compat.py @@ -0,0 +1,41 @@ +import copy +import json +import os +import sys + +from keystonelight import logging +from keystonelight import models +from keystonelight import test +from keystonelight import utils + + +KEYSTONECLIENT_REPO = 'git://github.com/4P/python-keystoneclient.git' + + + +class CompatTestCase(test.TestCase): + def setUp(self): + super(CompatTestCase, self).setUp() + + +class DiabloCompatTestCase(CompatTestCase): + def setUp(self): + revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') + self.add_path(revdir) + from keystoneclient.v2_0 import client as ks_client + reload(ks_client) + + self.app = self.loadapp('keystone_compat_diablo') + self.options = self.appconfig('keystone_compat_diablo') + + self.identity_backend = utils.import_object( + self.options['identity_driver'], options=self.options) + self.token_backend = utils.import_object( + self.options['token_driver'], options=self.options) + self.catalog_backend = utils.import_object( + self.options['catalog_driver'], options=self.options) + + super(DiabloCompatTestCase, self).setUp() + + def test_pass(self): + pass From 2ac753edfd662d0e5a24531f47c8f16feab16de8 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 31 Oct 2011 11:52:37 -0700 Subject: [PATCH 019/334] everything but the catalog --- keystonelight/backends/kvs.py | 4 ++++ keystonelight/identity.py | 3 +++ keystonelight/keystone_compat.py | 12 +++++++++--- keystonelight/service.py | 1 - keystonelight/test.py | 6 ++++++ tests/keystone_compat_diablo.conf | 2 ++ tests/test_keystoneclient_compat.py | 26 ++++++++++++++++++++++++++ 7 files changed, 50 insertions(+), 4 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 6dfa2c455f..a9221124b6 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -31,6 +31,10 @@ class KvsIdentity(object): tenant_ref = self.db.get('tenant-%s' % tenant_id) return tenant_ref + def get_tenant_by_name(self, tenant_name): + tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + return tenant_ref + def get_user(self, user_id): user_ref = self.db.get('user-%s' % user_id) return user_ref diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 9270719143..f63b55553a 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -25,3 +25,6 @@ class Manager(object): def get_tenant(self, context, tenant_id): return self.driver.get_tenant(tenant_id) + + def get_tenant_by_name(self, context, tenant_name): + return self.driver.get_tenant_by_name(tenant_name) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 7a9d16e734..403fac7452 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -67,13 +67,19 @@ class KeystoneController(service.BaseApplication): if 'passwordCredentials' in auth: username = auth['passwordCredentials'].get('username', '') password = auth['passwordCredentials'].get('password', '') - tenant = auth.get('tenantName', None) + tenant_name = auth.get('tenantName', None) + + # more compat + if tenant_name: + tenant_id = self.identity_api.get_tenant_by_name(tenant_name) + else: + tenant_id = auth.get('tenantId', None) (user_ref, tenant_ref, extras_ref) = \ self.identity_api.authenticate(context=context, user_id=username, password=password, - tenant_id=tenant) + tenant_id=tenant_id) token_ref = self.token_api.create_token(context, dict(expires='', user=user_ref, @@ -96,7 +102,7 @@ class KeystoneController(service.BaseApplication): assert tenant in user_ref['tenants'] tenant_ref = self.identity_api.get_tenant(context=context, - tenant_id=tenant) + tenant_id=tenant_id) extras_ref = self.identity_api.get_extras( context=context, user_id=user_ref['id'], diff --git a/keystonelight/service.py b/keystonelight/service.py index 78b6b8ecda..c607f8aa7c 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -94,7 +94,6 @@ class JsonBodyMiddleware(wsgi.Middleware): params_json = request.body if not params_json: - print "ASDASDASDASD" return params_parsed = json.loads(params_json) diff --git a/keystonelight/test.py b/keystonelight/test.py index 0c22f3777b..570b9ecc46 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -83,6 +83,12 @@ class TestCase(unittest.TestCase): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) return deploy.appconfig(config) + def serveapp(self, config): + app = self.loadapp(config) + server = wsgi.Server() + server.start(app, 0, key='socket') + return server + def client(self, app, *args, **kw): return TestClient(app, *args, **kw) diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index d2fa70c589..36c88964fb 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -2,6 +2,8 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +public_port = 8080 + [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index b94d60448a..2ef470de0f 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -35,7 +35,33 @@ class DiabloCompatTestCase(CompatTestCase): self.catalog_backend = utils.import_object( self.options['catalog_driver'], options=self.options) + self.server = self.serveapp('keystone_compat_diablo') + + self.tenant_bar = self.identity_backend._create_tenant( + 'bar', + models.Tenant(id='bar', name='BAR')) + + self.user_foo = self.identity_backend._create_user( + 'foo', + models.User(id='foo', + name='FOO', + tenants=[self.tenant_bar['id']], + password='foo')) + + self.extras_bar_foo = self.identity_backend._create_extras( + self.user_foo['id'], self.tenant_bar['id'], + dict(roles=[], + roles_links=[])) + super(DiabloCompatTestCase, self).setUp() def test_pass(self): + from keystoneclient.v2_0 import client as ks_client + + port = self.server.socket_info['socket'][1] + client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, + username='foo', + password='foo', + project_id='bar') + client.authenticate() pass From 4ba33be08e2ef24f61ec9a840df3f28cdb569818 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 13:35:27 -0700 Subject: [PATCH 020/334] add templated catalog backend Fix a few other issues, and adds passive aggressive comments and documentation. --- keystonelight/backends/templated.py | 81 +++++++++++++++++++++++++ keystonelight/keystone_compat.py | 5 +- keystonelight/test.py | 9 +++ tests/keystone_compat_diablo.conf | 3 +- tests/keystoneclient_compat_master.conf | 31 ++++++++++ tests/test_keystone_compat.py | 11 +++- tests/test_keystoneclient_compat.py | 11 ++-- 7 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 keystonelight/backends/templated.py create mode 100644 tests/keystoneclient_compat_master.conf diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py new file mode 100644 index 0000000000..e8406dc06e --- /dev/null +++ b/keystonelight/backends/templated.py @@ -0,0 +1,81 @@ +class TemplatedCatalog(object): + """A backend that generates endpoints for the Catalog based on templates. + + It is usually configured via config entries that look like: + + catalog.$REGION.$SERVICE.$key = $value + + and is stored in a similar looking hierarchy. Where a value can contain + values to be interpolated by standard python string interpolation that look + like (the % is replaced by a $ due to paste attmepting to interpolate on its + own: + + http://localhost:$(public_port)s/ + + When expanding the template it will pass in a dict made up of the options + instance plus a few additional key-values, notably tenant_id and user_id. + + It does not care what the keys and values are but it is worth noting that + keystone_compat will expect certain keys to be there so that it can munge + them into the output format keystone expects. These keys are: + + name - the name of the service, most likely repeated for all services of + the same type, across regions. + adminURL - the url of the admin endpoint + publicURL - the url of the public endpoint + internalURL - the url of the internal endpoint + + """ + + def __init__(self, options, templates=None): + self.options = options + + if templates: + self.templates = templates + else: + self._load_templates(options) + + + def _load_templates(self, options): + o = {} + for k, v in options.iteritems(): + if not k.startswith('catalog.'): + continue + + parts = k.split('.') + + region = parts[1] + service = parts[2] + key = parts[3] + + region_ref = o.get(region, {}) + service_ref = region_ref.get(service, {}) + service_ref[key] = v + + region_ref[service] = service_ref + o[region] = region_ref + + self.templates = o + + def get_catalog(self, user_id, tenant_id, extras=None): + d = self.options.copy() + d.update({'tenant_id': tenant_id, + 'user_id': user_id}) + + o = {} + for region, region_ref in self.templates.iteritems(): + o[region] = {} + for service, service_ref in region_ref.iteritems(): + o[region][service] = {} + for k, v in service_ref.iteritems(): + v = v.replace('$(', '%(') + o[region][service][k] = v % d + + return o + + + + + + + diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 403fac7452..22451b4b70 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -71,7 +71,10 @@ class KeystoneController(service.BaseApplication): # more compat if tenant_name: - tenant_id = self.identity_api.get_tenant_by_name(tenant_name) + tenant_ref = self.identity_api.get_tenant_by_name( + context=context, tenant_name=tenant_name) + tenant_id = tenant_ref['id'] + logging.debug('RENANT: %s', tenant_ref) else: tenant_id = auth.get('tenantId', None) diff --git a/keystonelight/test.py b/keystonelight/test.py index 570b9ecc46..9f28b1df31 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -1,3 +1,4 @@ +import ConfigParser import os import unittest import sys @@ -67,6 +68,9 @@ class TestCase(unittest.TestCase): super(TestCase, self).__init__(*args, **kw) self._paths = [] + def setUp(self): + super(TestCase, self).setUp() + def tearDown(self): for path in self._paths: if path in sys.path: @@ -87,6 +91,11 @@ class TestCase(unittest.TestCase): app = self.loadapp(config) server = wsgi.Server() server.start(app, 0, key='socket') + + # Service catalog tests need to know the port we ran on. + options = self.appconfig(config) + port = server.socket_info['socket'][1] + options['public_port'] = port return server def client(self, app, *args, **kw): diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index 36c88964fb..8318a0ca7c 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -2,8 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -public_port = 8080 - +public_port = 5000 [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf new file mode 100644 index 0000000000..e0595b8c4b --- /dev/null +++ b/tests/keystoneclient_compat_master.conf @@ -0,0 +1,31 @@ +[DEFAULT] +catalog_driver = keystonelight.backends.templated.TemplatedCatalog +identity_driver = keystonelight.backends.kvs.KvsIdentity +token_driver = keystonelight.backends.kvs.KvsToken +public_port = 5000 + +# config for TemplatedCatalog, using camelCase because I don't want to do +# translations for keystone compat +catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s +catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s +catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s + + +# in old config: +# first 1 == public +# second 1 == enabled + +[filter:debug] +paste.filter_factory = keystonelight.wsgi:Debug.factory + +[filter:token_auth] +paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory + +[filter:json_body] +paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory + +[app:keystone] +paste.app_factory = keystonelight.keystone_compat:app_factory + +[pipeline:main] +pipeline = token_auth json_body debug keystone diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 2d956f1d36..3577122b68 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -19,6 +19,15 @@ KEYSTONE_SAMPLE_DIR = 'keystone/content/common/samples' class CompatTestCase(test.TestCase): + """Test compatibility against various versions of keystone's docs. + + It should be noted that the docs for any given revision have rarely, if ever, + reflected the actual usage or reliable sample output of the system, so these + tests are largely a study of frustration and its effects on developer + productivity. + + """ + def setUp(self): super(CompatTestCase, self).setUp() @@ -120,7 +129,7 @@ class DiabloCompatTestCase(CompatTestCase): {'auth': {'passwordCredentials': {'username': self.user_123['id'], 'password': self.user_123['password'], }, - 'tenantName': self.tenant_345['id']}}) + 'tenantName': self.tenant_345['name']}}) resp = client.post('/v2.0/tokens', body=post_data) data = json.loads(resp.body) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 2ef470de0f..11e32812f7 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -18,15 +18,17 @@ class CompatTestCase(test.TestCase): super(CompatTestCase, self).setUp() -class DiabloCompatTestCase(CompatTestCase): +class MasterCompatTestCase(CompatTestCase): def setUp(self): + super(MasterCompatTestCase, self).setUp() + revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') self.add_path(revdir) from keystoneclient.v2_0 import client as ks_client reload(ks_client) - self.app = self.loadapp('keystone_compat_diablo') - self.options = self.appconfig('keystone_compat_diablo') + self.app = self.loadapp('keystoneclient_compat_master') + self.options = self.appconfig('keystoneclient_compat_master') self.identity_backend = utils.import_object( self.options['identity_driver'], options=self.options) @@ -35,7 +37,7 @@ class DiabloCompatTestCase(CompatTestCase): self.catalog_backend = utils.import_object( self.options['catalog_driver'], options=self.options) - self.server = self.serveapp('keystone_compat_diablo') + self.server = self.serveapp('keystoneclient_compat_master') self.tenant_bar = self.identity_backend._create_tenant( 'bar', @@ -53,7 +55,6 @@ class DiabloCompatTestCase(CompatTestCase): dict(roles=[], roles_links=[])) - super(DiabloCompatTestCase, self).setUp() def test_pass(self): from keystoneclient.v2_0 import client as ks_client From 3caf2a8db462c512f25b76f0074113daa0bbb7e3 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 14:07:56 -0700 Subject: [PATCH 021/334] remove test_keystone_compat's catalog tests the sample data is clearly insane --- keystonelight/backends/templated.py | 1 - keystonelight/keystone_compat.py | 38 ++++++++++++++++++++++++++++- tests/test_keystone_compat.py | 19 +++++++++------ tests/test_keystoneclient_compat.py | 2 +- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index e8406dc06e..99e867b1df 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -35,7 +35,6 @@ class TemplatedCatalog(object): else: self._load_templates(options) - def _load_templates(self, options): o = {} for k, v in options.iteritems(): diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 22451b4b70..8a669afe6d 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -125,9 +125,45 @@ class KeystoneController(service.BaseApplication): def _format_authenticate(self, token_ref, catalog_ref): o = self._format_token(token_ref) - o['access']['serviceCatalog'] = catalog_ref + o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) return o + def _format_catalog(self, catalog_ref): + """KeystoneLight catalogs look like: + + {$REGION: { + {$SERVICE: { + $key1: $value1, + ... + } + } + } + + Keystone's look like + + [{'name': $SERVICE[name], + 'type': $SERVICE, + 'endpoints': [{ + 'tenantId': $tenant_id, + ... + 'region': $REGION, + }], + 'endpoints_links': [], + }] + + """ + if not catalog_ref: + return {} + + o = [] + services = {} + for region, region_ref in catalog_ref.iteritems(): + for service, service_ref in region_ref.iteritems(): + new_service_ref = services.get(service, {}) + new_service_ref['name'] = service_ref['name'] + + + #admin-only def validate_token(self, context, token_id, belongs_to=None): """Check that a token is valid. diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 3577122b68..609d3a6427 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -72,12 +72,13 @@ class CompatTestCase(test.TestCase): # NOTE(termie): the service catalog in the sample doesn't really have # anything to do with the auth being returned, so just load # it fully from a fixture and add it to our db - catalog = json.load(open( - os.path.join(os.path.dirname(__file__), - 'keystone_compat_diablo_sample_catalog.json'))) - self.catalog_backend._create_catalog(self.user_123['id'], - self.tenant_345['id'], - catalog) + # NOTE(termie): actually all the data is insane anyway, so don't bother + #catalog = json.load(open( + # os.path.join(os.path.dirname(__file__), + # 'keystone_compat_diablo_sample_catalog.json'))) + #self.catalog_backend._create_catalog(self.user_123['id'], + # self.tenant_345['id'], + # catalog) # tenants_for_token call self.user_foo = self.identity_backend._create_user( @@ -137,8 +138,10 @@ class DiabloCompatTestCase(CompatTestCase): self.assert_('expires' in data['access']['token']) self.assertDeepEquals(self.auth_response['access']['user'], data['access']['user']) - self.assertDeepEquals(self.auth_response['access']['serviceCatalog'], - data['access']['serviceCatalog']) + # there is pretty much no way to generate sane data that corresponds to + # the sample data + #self.assertDeepEquals(self.auth_response['access']['serviceCatalog'], + # data['access']['serviceCatalog']) def test_validate_token_scoped(self): client = self.client(self.app, token=self.token_123['id']) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 11e32812f7..d0c54c0c0e 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -56,7 +56,7 @@ class MasterCompatTestCase(CompatTestCase): roles_links=[])) - def test_pass(self): + def test_authenticate(self): from keystoneclient.v2_0 import client as ks_client port = self.server.socket_info['socket'][1] From 2f2465eafa483c1ce365bef01e56211788c8c29d Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 14:28:28 -0700 Subject: [PATCH 022/334] working authenticate in keystoneclient --- keystonelight/identity.py | 3 +++ keystonelight/keystone_compat.py | 32 ++++++++++++++++++------- tests/keystoneclient_compat_master.conf | 1 + 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index f63b55553a..0d8bde22fd 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -28,3 +28,6 @@ class Manager(object): def get_tenant_by_name(self, context, tenant_name): return self.driver.get_tenant_by_name(tenant_name) + + def get_extras(self, context, user_id, tenant_id): + return self.driver.get_extras(user_id, tenant_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 8a669afe6d..86746d74eb 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -74,7 +74,6 @@ class KeystoneController(service.BaseApplication): tenant_ref = self.identity_api.get_tenant_by_name( context=context, tenant_name=tenant_name) tenant_id = tenant_ref['id'] - logging.debug('RENANT: %s', tenant_ref) else: tenant_id = auth.get('tenantId', None) @@ -94,22 +93,31 @@ class KeystoneController(service.BaseApplication): tenant_id=tenant_ref['id'], extras=extras_ref) - elif 'tokenCredentials' in auth: - token = auth['tokenCredentials'].get('token', None) - tenant = auth.get('tenantName') + elif 'token' in auth: + token = auth['token'].get('id', None) + + tenant_name = auth.get('tenantName') + + # more compat + if tenant_name: + tenant_ref = self.identity_api.get_tenant_by_name( + context=context, tenant_name=tenant_name) + tenant_id = tenant_ref['id'] + else: + tenant_id = auth.get('tenantId', None) old_token_ref = self.token_api.get_token(context=context, token_id=token) user_ref = old_token_ref['user'] - assert tenant in user_ref['tenants'] + assert tenant_id in user_ref['tenants'] tenant_ref = self.identity_api.get_tenant(context=context, tenant_id=tenant_id) extras_ref = self.identity_api.get_extras( context=context, user_id=user_ref['id'], - tenant_id=tenant_ref['tenant']['id']) + tenant_id=tenant_ref['id']) token_ref = self.token_api.create_token(context, dict(expires='', user=user_ref, @@ -155,14 +163,22 @@ class KeystoneController(service.BaseApplication): if not catalog_ref: return {} - o = [] services = {} for region, region_ref in catalog_ref.iteritems(): for service, service_ref in region_ref.iteritems(): new_service_ref = services.get(service, {}) - new_service_ref['name'] = service_ref['name'] + new_service_ref['name'] = service_ref.pop('name') + new_service_ref['type'] = service + new_service_ref['endpoints_links'] = [] + service_ref['region'] = region + endpoints_ref = new_service_ref.get('endpoints', []) + endpoints_ref.append(service_ref) + new_service_ref['endpoints'] = endpoints_ref + services[service] = new_service_ref + + return services.values() #admin-only def validate_token(self, context, token_id, belongs_to=None): diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index e0595b8c4b..f274767313 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -9,6 +9,7 @@ public_port = 5000 catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s +catalog.RegionOne.identity.name = 'Identity Service' # in old config: From 0d4e11c48a9082ccc6c56aee2643c87f28367634 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 15:04:01 -0700 Subject: [PATCH 023/334] authenticate and tenants working --- keystonelight/backends/templated.py | 4 ++++ keystonelight/test.py | 21 +++++++++++++++++++-- keystonelight/wsgi.py | 1 + tests/keystoneclient_compat_master.conf | 6 +++--- tests/test_keystoneclient_compat.py | 15 +++++++++++++-- 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index 99e867b1df..1a848c73a6 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -1,3 +1,6 @@ +from keystonelight import logging + + class TemplatedCatalog(object): """A backend that generates endpoints for the Catalog based on templates. @@ -29,6 +32,7 @@ class TemplatedCatalog(object): def __init__(self, options, templates=None): self.options = options + logging.debug('CATALOG PUBLIC PORT BEFORE: %s', options['public_port']) if templates: self.templates = templates diff --git a/keystonelight/test.py b/keystonelight/test.py index 9f28b1df31..a710203f11 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -5,6 +5,7 @@ import sys from paste import deploy +from keystonelight import logging from keystonelight import utils from keystonelight import wsgi @@ -67,6 +68,7 @@ class TestCase(unittest.TestCase): def __init__(self, *args, **kw): super(TestCase, self).__init__(*args, **kw) self._paths = [] + self._memo = {} def setUp(self): super(TestCase, self).setUp() @@ -93,11 +95,26 @@ class TestCase(unittest.TestCase): server.start(app, 0, key='socket') # Service catalog tests need to know the port we ran on. - options = self.appconfig(config) port = server.socket_info['socket'][1] - options['public_port'] = port + self._update_server_options(server, 'public_port', port) + logging.debug('PUBLIC PORT: %s', app.options['public_port']) return server + def _update_server_options(self, server, key, value): + """Hack to allow us to make changes to the options used by backends. + + A possible better solution would be to have a global config registry. + + """ + last = server + while hasattr(last, 'application') or hasattr(last, 'options'): + logging.debug('UPDATE %s', last.__class__) + if hasattr(last, 'options'): + last.options[key] = value + if not hasattr(last, 'application'): + break + last = last.application + def client(self, app, *args, **kw): return TestClient(app, *args, **kw) diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 5be3170e98..4f3c654417 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -58,6 +58,7 @@ class Server(object): def start(self, application, port, host='0.0.0.0', key=None, backlog=128): """Run a WSGI server with the given application.""" + self.application = application arg0 = sys.argv[0] logging.debug('Starting %(arg0)s on %(host)s:%(port)s' % locals()) socket = eventlet.listen((host, port), backlog=backlog) diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index f274767313..049759171a 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -6,9 +6,9 @@ public_port = 5000 # config for TemplatedCatalog, using camelCase because I don't want to do # translations for keystone compat -catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s -catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s -catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0/$(tenant_id)s +catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 catalog.RegionOne.identity.name = 'Identity Service' diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index d0c54c0c0e..1f73adb2fc 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -55,14 +55,25 @@ class MasterCompatTestCase(CompatTestCase): dict(roles=[], roles_links=[])) + #def test_authenticate(self): + # from keystoneclient.v2_0 import client as ks_client - def test_authenticate(self): + # port = self.server.socket_info['socket'][1] + # client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, + # username='foo', + # password='foo', + # project_id='bar') + # client.authenticate() + + def test_authenticate_and_tenants(self): from keystoneclient.v2_0 import client as ks_client port = self.server.socket_info['socket'][1] + self.options['public_port'] = port client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, username='foo', password='foo', project_id='bar') client.authenticate() - pass + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) From 169c4fb8c0cfcb0e03dc6208715e8ea21c9dd0ee Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 16:39:54 -0700 Subject: [PATCH 024/334] updated readme --- README.rst | 87 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index c6b57552f1..d74ecb6716 100644 --- a/README.rst +++ b/README.rst @@ -1,31 +1,82 @@ +Keystone *Light* +================ + +*Always Smooth* + +Keystone Light is a re-interpretation of the Keystone project that provides +Identity, Token and Catalog services for use specifically by projects in the +OpenStack family. -General expected data model: +---------- +Data Model +---------- - ( tenants >--< users ) --< roles +Keystone Light was designed from the ground up to be amenable to multiple +styles of backends and as such many of the methods and data types will happily +accept more data than they know what to do with and pass them on to a backend. - Tenants and Users have a many-to-many relationship. - A given Tenant-User pair can have many Roles. +There are a few main data types: + + * **User**: has account credentials, is associated with one or more tenants + * **Tenant**: unit of ownership in openstack, contains one or more users + * **Token**: identifying credential associated with a user or user and tenant + * **Extras**: bucket of key-values associated with a user-tenant pair, + typically used to define roles + +While the general data model allows a many-to-many relationship between Users +and Tenants and a many-to-one relationship between Extras and User-Tenant pairs, +the actual backend implementations take varying levels of advantage of that +functionality. -Tenant Schema: - id: something big and unique - name: something displayable - .. created_at: datetime - .. deleted_at: datetime +KVS Backend +----------- -User Schema: - id: something big and unique - name: something displayable - .. created_at: datetime - .. deleted_at: datetime +A simple backend interface meant to be further backended on anything that can +support primary key lookups, the most trivial implementation being an in-memory +dict. + +Supports all features of the general data model. -General service model: +PAM Backend +----------- - (1) a web service with an API - (2) a variety of backend storage schemes for tenant-user pairs. - (3) a simple token datastore +Extra simple backend that uses the current system's PAM service to authenticate, +providing a one-to-one relationship between Users and Tenants with the `root` +User also having the 'admin' role. +Templated Backend +----------------- +Largely designed for a common use case around service catalogs in the Keystone +project, a Catalog backend that simply expands pre-configured templates to +provide catalog data. + + +--------------- +Keystone Compat +--------------- + +While Keystone Light takes a fundamentally different approach to its services +from Keystone, a compatibility layer is included to make switching much easier +for projects already attempting to use Keystone. + +The compatibility service is activated by defining an alternate application in +the paste.deploy config and adding it to your main pipeline:: + + [app:keystone] + paste.app_factory = keystonelight.keystone_compat:app_factory + + +----------- +Still To Do +----------- + +* Dev and testing setups would do well with some user/tenant/etc CRUD, for the + KVS backends at least. +* Fixture loading functionality would also be killer tests and dev. +* LDAP backend. +* Keystone import. From 1d1db0f7c0ae338463a6b56cf17580322587e849 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 16:41:01 -0700 Subject: [PATCH 025/334] rst blah blah --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d74ecb6716..03bb39d326 100644 --- a/README.rst +++ b/README.rst @@ -21,8 +21,7 @@ There are a few main data types: * **User**: has account credentials, is associated with one or more tenants * **Tenant**: unit of ownership in openstack, contains one or more users * **Token**: identifying credential associated with a user or user and tenant - * **Extras**: bucket of key-values associated with a user-tenant pair, - typically used to define roles + * **Extras**: bucket of key-values associated with a user-tenant pair, typically used to define roles. While the general data model allows a many-to-many relationship between Users and Tenants and a many-to-one relationship between Extras and User-Tenant pairs, From d3cc7983a168366cd262accec3cfc98f74317836 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Nov 2011 16:43:47 -0700 Subject: [PATCH 026/334] add example --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 03bb39d326..94818ad434 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,15 @@ Largely designed for a common use case around service catalogs in the Keystone project, a Catalog backend that simply expands pre-configured templates to provide catalog data. +Example paste.deploy config (uses $ instead of % to avoid ConfigParser's +interpolation):: + + [DEFAULT] + catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.name = 'Identity Service' + --------------- Keystone Compat From f8ec4f6d6a307d21cd9c8d6a6ab7ebfdf7897173 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:08:38 -0700 Subject: [PATCH 027/334] add some todo --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 94818ad434..b424a1d42c 100644 --- a/README.rst +++ b/README.rst @@ -83,8 +83,9 @@ the paste.deploy config and adding it to your main pipeline:: Still To Do ----------- -* Dev and testing setups would do well with some user/tenant/etc CRUD, for the - KVS backends at least. -* Fixture loading functionality would also be killer tests and dev. -* LDAP backend. -* Keystone import. + * Dev and testing setups would do well with some user/tenant/etc CRUD, for the + KVS backends at least. + * Fixture loading functionality would also be killer tests and dev. + * LDAP backend. + * Keystone import. + * Admin-only interface From 4b48845c221839efd6a22b11fb498ece2bfebbc2 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:09:17 -0700 Subject: [PATCH 028/334] minor whitespace cleanup --- keystonelight/keystone_compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 86746d74eb..cf99244c04 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -17,7 +17,6 @@ class KeystoneRouter(wsgi.Router): self.options = options self.keystone_controller = KeystoneController(options) - mapper = routes.Mapper() mapper.connect('/v2.0/tokens', controller=self.keystone_controller, From b514897be6fc1334b7cfc4d29137563286c2912c Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:10:13 -0700 Subject: [PATCH 029/334] add a default conf --- keystonelight/service.py | 5 +++++ tests/default.conf | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/default.conf diff --git a/keystonelight/service.py b/keystonelight/service.py index c607f8aa7c..1db788698a 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -198,3 +198,8 @@ def identity_app_factory(global_conf, **local_conf): conf.update(local_conf) return Router(conf) + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return Router(conf) diff --git a/tests/default.conf b/tests/default.conf new file mode 100644 index 0000000000..fa6ac28b52 --- /dev/null +++ b/tests/default.conf @@ -0,0 +1,19 @@ +[DEFAULT] +catalog_driver = keystonelight.backends.kvs.KvsCatalog +identity_driver = keystonelight.backends.kvs.KvsIdentity +token_driver = keystonelight.backends.kvs.KvsToken + +[filter:debug] +paste.filter_factory = keystonelight.wsgi:Debug.factory + +[filter:token_auth] +paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory + +[filter:json_body] +paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory + +[app:keystonelight] +paste.app_factory = keystonelight.service:app_factory + +[pipeline:main] +pipeline = token_auth json_body debug keystonelight From 4c8a5ac747aa6165bfa1a3f30e93491f0cda730a Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:11:36 -0700 Subject: [PATCH 030/334] add some failing tests --- keystonelight/backends/kvs.py | 8 ++++++ tests/test_backend_kvs.py | 46 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tests/test_backend_kvs.py diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index a9221124b6..3d1f53c445 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -12,10 +12,18 @@ class KvsIdentity(object): def __init__(self, options, db=None): if db is None: db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) self.db = db # Public interface def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. + + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. + + """ user_ref = self.get_user(user_id) tenant_ref = None extras_ref = None diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py new file mode 100644 index 0000000000..b67cf57547 --- /dev/null +++ b/tests/test_backend_kvs.py @@ -0,0 +1,46 @@ + +from keystonelight import models +from keystonelight import test +from keystonelight.backends import kvs + + +class KvsIdentity(test.TestCase): + def setUp(self): + super(KvsIdentity, self).setUp() + options = self.appconfig('default') + self.identity_api = kvs.KvsIdentity(options=options, db={}) + self._load_fixtures() + + def _load_fixtures(self): + self.tenant_bar = self.identity_api._create_tenant( + 'bar', + models.Tenant(id='bar', name='BAR')) + self.user_foo = self.identity_api._create_user( + 'foo', + models.User(id='foo', + name='FOO', + password='foo2', + tenants=[self.tenant_bar['id']])) + + + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') + + + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) From 912c2227031f4da30ffb04b6daa9349951bb3c11 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:24:51 -0700 Subject: [PATCH 031/334] add some tests and get others to pass --- keystonelight/backends/kvs.py | 9 +++++---- keystonelight/test.py | 17 +++++++++++------ tests/test_backend_kvs.py | 22 ++++++++++++++++++++-- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 3d1f53c445..7bc338b11d 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -27,12 +27,13 @@ class KvsIdentity(object): user_ref = self.get_user(user_id) tenant_ref = None extras_ref = None - if user_ref['password'] != password: + if not user_ref or user_ref.get('password') != password: raise AssertionError('Invalid user / password') + if tenant_id and tenant_id not in user_ref['tenants']: + raise AssertionError('Invalid tenant') - if tenant_id and tenant_id in user_ref['tenants']: - tenant_ref = self.get_tenant(tenant_id) - extras_ref = self.get_extras(user_id, tenant_id) + tenant_ref = self.get_tenant(tenant_id) + extras_ref = self.get_extras(user_id, tenant_id) return (user_ref, tenant_ref, extras_ref) def get_tenant(self, tenant_id): diff --git a/keystonelight/test.py b/keystonelight/test.py index a710203f11..9a24883c67 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -1,6 +1,8 @@ import ConfigParser +import logging import os import unittest +import subprocess import sys from paste import deploy @@ -24,13 +26,16 @@ def checkout_vendor(repo, rev): name = name[:-4] revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) + try: + if not os.path.exists(revdir): + utils.git('clone', repo, revdir) - if not os.path.exists(revdir): - utils.git('clone', repo, revdir) - - cd(revdir) - utils.git('pull') - utils.git('checkout', '-q', rev) + cd(revdir) + utils.git('pull') + utils.git('checkout', '-q', rev) + except subprocess.CalledProcessError as e: + logging.warning('Failed to checkout %s', repo) + pass return revdir diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index b67cf57547..8eb1ae4a30 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -21,7 +21,9 @@ class KvsIdentity(test.TestCase): name='FOO', password='foo2', tenants=[self.tenant_bar['id']])) - + self.extras_foobar = self.identity_api._create_extras( + 'foo', 'bar', + {'extra': 'extra'}) def test_authenticate_bad_user(self): self.assertRaises(AssertionError, @@ -37,10 +39,26 @@ class KvsIdentity(test.TestCase): tenant_id=self.tenant_bar['id'], password=self.user_foo['password'] + 'WRONG') - def test_authenticate_invalid_tenant(self): self.assertRaises(AssertionError, self.identity_api.authenticate, user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id'] + 'WRONG', password=self.user_foo['password']) + + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(extras_ref is None) + + def test_authenticate(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(extras_ref, self.extras_foobar) From 3f0137aeaea9f9832fc43d86e159cf2131893eec Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:32:48 -0700 Subject: [PATCH 032/334] test the other methods too --- tests/test_backend_kvs.py | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 8eb1ae4a30..718c047879 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -62,3 +62,49 @@ class KvsIdentity(test.TestCase): self.assertDictEquals(user_ref, self.user_foo) self.assertDictEquals(tenant_ref, self.tenant_bar) self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) + + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + self.assertDictEquals(user_ref, self.user_foo) + + def test_get_extras_bad_user(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(extras_ref is None) + + def test_get_extras_bad_tenant(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(extras_ref is None) + + def test_get_extras(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(extras_ref, self.extras_foobar) From f86bf25f3275a43d438409ffb4213beb70ecc673 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:41:21 -0700 Subject: [PATCH 033/334] added tests for tokens --- keystonelight/backends/kvs.py | 4 ++++ tests/test_backend_kvs.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 7bc338b11d..e7f959b1ac 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -70,6 +70,8 @@ class KvsToken(object): def __init__(self, options, db=None): if db is None: db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) self.db = db # Public interface @@ -88,6 +90,8 @@ class KvsCatalog(object): def __init__(self, options, db=None): if db is None: db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) self.db = db # Public interface diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 718c047879..78575c3aaa 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -1,3 +1,4 @@ +import uuid from keystonelight import models from keystonelight import test @@ -108,3 +109,24 @@ class KvsIdentity(test.TestCase): user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id']) self.assertDictEquals(extras_ref, self.extras_foobar) + + +class KvsToken(test.TestCase): + def setUp(self): + super(KvsToken, self).setUp() + options = self.appconfig('default') + self.token_api = kvs.KvsToken(options=options, db={}) + + def test_token_crud(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, + 'a': 'b'} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + + new_data_ref = self.token_api.get_token(token_id) + self.assertEquals(new_data_ref, data) + + self.token_api.delete_token(token_id) + deleted_data_ref = self.token_api.get_token(token_id) + self.assert_(deleted_data_ref is None) From 344d21ca699eb87c7fa9086c67a43758d60db8bf Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 3 Nov 2011 14:48:50 -0700 Subject: [PATCH 034/334] added catalog tests --- keystonelight/backends/kvs.py | 5 +++-- tests/test_backend_kvs.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index e7f959b1ac..cbf17297f4 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -96,8 +96,9 @@ class KvsCatalog(object): # Public interface def get_catalog(self, user_id, tenant_id, extras=None): - return self.db.get('catalog-%s' % tenant_id) + return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) # Private interface def _create_catalog(self, user_id, tenant_id, data): - self.db.set('catalog-%s' % tenant_id, data) + self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) + return data diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 78575c3aaa..3561a13c5b 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -130,3 +130,28 @@ class KvsToken(test.TestCase): self.token_api.delete_token(token_id) deleted_data_ref = self.token_api.get_token(token_id) self.assert_(deleted_data_ref is None) + + +class KvsCatalog(test.TestCase): + def setUp(self): + super(KvsCatalog, self).setUp() + options = self.appconfig('default') + self.catalog_api = kvs.KvsCatalog(options=options, db={}) + self._load_fixtures() + + def _load_fixtures(self): + self.catalog_foobar = self.catalog_api._create_catalog( + 'foo', 'bar', + {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) + + def test_get_catalog_bad_user(self): + catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') + self.assert_(catalog_ref is None) + + def test_get_catalog_bad_tenant(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') + self.assert_(catalog_ref is None) + + def test_get_catalog(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar') + self.assertDictEquals(catalog_ref, self.catalog_foobar) From 002ae338be2d9dd8ba742e322bd97c732f3e0e60 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 4 Nov 2011 15:17:18 -0700 Subject: [PATCH 035/334] whitespace --- keystonelight/backends/kvs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index cbf17297f4..caa6e4c3dc 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -6,8 +6,10 @@ class DictKvs(dict): def delete(self, key): del self[key] + INMEMDB = DictKvs() + class KvsIdentity(object): def __init__(self, options, db=None): if db is None: From 6cb7e6cb337844ecd265727302108354becadccb Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 10:38:40 -0800 Subject: [PATCH 036/334] link diagrams --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 94818ad434..d78646819a 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,14 @@ the paste.deploy config and adding it to your main pipeline:: [app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory +Also relevant to Keystone compatibility are these sequence diagrams (openable +with sdedit_) + +.. _sdedit: http://sourceforge.net/projects/sdedit/files/sdedit/4.0/ + +Diagram: keystone_compat_flows.sdx_ + +.. _: https://raw.github.com/termie/keystonelight/master/docs/keystone_compat_flows.sdx ----------- Still To Do From 29e4e54ce31292becc28d14012c09972efa70404 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 14:11:27 -0800 Subject: [PATCH 037/334] modify requirements --- README.rst | 1 + keystonelight/wsgi.py | 4 ---- tools/pip-requires | 1 - tools/pip-requires-test | 2 ++ 4 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 tools/pip-requires-test diff --git a/README.rst b/README.rst index 344be5a927..04b6956bc9 100644 --- a/README.rst +++ b/README.rst @@ -97,3 +97,4 @@ Still To Do * LDAP backend. * Keystone import. * Admin-only interface + * Don't check git checkouts as often, to speed up tests diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 4f3c654417..6eea8aa974 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -26,7 +26,6 @@ import sys import eventlet import eventlet.wsgi eventlet.patcher.monkey_patch(all=False, socket=True, time=True) -import hflags as flags import routes import routes.middleware import webob @@ -35,9 +34,6 @@ import webob.exc from paste import deploy -FLAGS = flags.FLAGS - - class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" diff --git a/tools/pip-requires b/tools/pip-requires index 30a8d58337..45cdf4523f 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,4 +1,3 @@ -hflags pam==0.1.4 WebOb==0.9.8 eventlet==0.9.12 diff --git a/tools/pip-requires-test b/tools/pip-requires-test new file mode 100644 index 0000000000..769396eb9b --- /dev/null +++ b/tools/pip-requires-test @@ -0,0 +1,2 @@ +# for python-keystoneclient +httplib2 From d17e1cf533e72d0912eeed76c133357da4b60123 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 14:13:35 -0800 Subject: [PATCH 038/334] remove italics on Light --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 04b6956bc9..3277d44468 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -Keystone *Light* -================ +Keystone Light +============== *Always Smooth* From 2bc437609d0ab1faa2e474f5710665fc427f74f9 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 14:34:13 -0800 Subject: [PATCH 039/334] add run_tests.sh and pep8 stuff --- bin/keystone | 4 +- keystonelight/backends/kvs.py | 1 - keystonelight/backends/templated.py | 7 - keystonelight/identity.py | 3 +- keystonelight/service.py | 3 +- keystonelight/test.py | 2 - keystonelight/utils.py | 1 + keystonelight/wsgi.py | 55 ----- run_tests.py | 360 ++++++++++++++++++++++++++++ run_tests.sh | 153 ++++++++++++ tools/install_venv.py | 132 ++++++++++ tools/pip-requires-test | 12 + tools/with_venv.sh | 4 + 13 files changed, 669 insertions(+), 68 deletions(-) create mode 100644 run_tests.py create mode 100755 run_tests.sh create mode 100644 tools/install_venv.py create mode 100755 tools/with_venv.sh diff --git a/bin/keystone b/bin/keystone index fcfa378a93..3f3620fa62 100755 --- a/bin/keystone +++ b/bin/keystone @@ -9,7 +9,9 @@ import sys possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'keystonelight', '__init__.py')): +if os.path.exists(os.path.join(possible_topdir, + 'keystonelight', + '__init__.py')): sys.path.insert(0, possible_topdir) import logging diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index caa6e4c3dc..09baa75922 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -1,4 +1,3 @@ - class DictKvs(dict): def set(self, key, value): self[key] = value diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index 1a848c73a6..e3e9b65895 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -75,10 +75,3 @@ class TemplatedCatalog(object): o[region][service][k] = v % d return o - - - - - - - diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 0d8bde22fd..a144e938f5 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -1,7 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # these will be the basic data types for tenants and users -# backends will make use of them to return something that conforms to their apis +# backends will make use of them to return something that conforms to their +# apis from keystonelight import utils diff --git a/keystonelight/service.py b/keystonelight/service.py index 1db788698a..799bd354af 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -135,7 +135,8 @@ class IdentityController(BaseApplication): self.options = options def authenticate(self, context, **kwargs): - tenant, user, extras = self.identity_api.authenticate(context, **kwargs) + tenant, user, extras = self.identity_api.authenticate(context, + **kwargs) token = self.token_api.create_token(context, dict(tenant=tenant, user=user, diff --git a/keystonelight/test.py b/keystonelight/test.py index 9a24883c67..e32a78ec3a 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -169,5 +169,3 @@ class TestCase(unittest.TestCase): except AssertionError as e: raise raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) - - diff --git a/keystonelight/utils.py b/keystonelight/utils.py index 1c006e1cc5..ea4404f3c4 100644 --- a/keystonelight/utils.py +++ b/keystonelight/utils.py @@ -43,6 +43,7 @@ def import_object(import_str, *args, **kw): cls = import_class(import_str) return cls(*args, **kw) + # From python 2.7 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 6eea8aa974..c31052c6f4 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -230,7 +230,6 @@ class Debug(Middleware): logging.debug(line) logging.debug('') - resp = req.get_response(self.application) logging.debug('%s %s %s', ('*' * 20), 'RESPONSE HEADERS', ('*' * 20)) @@ -308,57 +307,3 @@ class Router(object): return webob.exc.HTTPNotFound() app = match['controller'] return app - - -def paste_config_file(basename): - """Find the best location in the system for a paste config file. - - Search Order - ------------ - - The search for a paste config file honors `FLAGS.state_path`, which in a - version checked out from bzr will be the `nova` directory in the top level - of the checkout, and in an installation for a package for your distribution - will likely point to someplace like /etc/nova. - - This method tries to load places likely to be used in development or - experimentation before falling back to the system-wide configuration - in `/etc/nova/`. - - * Current working directory - * the `etc` directory under state_path, because when working on a checkout - from bzr this will point to the default - * top level of FLAGS.state_path, for distributions - * /etc/nova, which may not be diffrerent from state_path on your distro - - """ - configfiles = [basename, - os.path.join(FLAGS.state_path, 'etc', 'nova', basename), - os.path.join(FLAGS.state_path, 'etc', basename), - os.path.join(FLAGS.state_path, basename), - '/etc/nova/%s' % basename] - for configfile in configfiles: - if os.path.exists(configfile): - return configfile - - -def load_paste_configuration(filename, appname): - """Returns a paste configuration dict, or None.""" - filename = os.path.abspath(filename) - config = None - try: - config = deploy.appconfig('config:%s' % filename, name=appname) - except LookupError: - pass - return config - - -def load_paste_app(filename, appname): - """Builds a wsgi app from a paste config, None if app not configured.""" - filename = os.path.abspath(filename) - app = None - try: - app = deploy.loadapp('config:%s' % filename, name=appname) - except LookupError: - pass - return app diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 0000000000..2120373791 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + +import gettext +import heapq +import logging +import os +import unittest +import sys +import time + +from nose import config +from nose import core +from nose import result + + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + + +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + +class NovaTestResult(result.TextTestResult): + def __init__(self, *args, **kw): + self.show_elapsed = kw.pop('show_elapsed') + result.TextTestResult.__init__(self, *args, **kw) + self.num_slow_tests = 5 + self.slow_tests = [] # this is a fixed-sized heap + self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout + + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, + # _handleElapsedTime will fail, causing the wrong error message to + # be outputted. + self.start_time = time.time() + + def getDescription(self, test): + return str(test) + + def _handleElapsedTime(self, test): + self.elapsed_time = time.time() - self.start_time + item = (self.elapsed_time, test) + # Record only the n-slowest tests using heap + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + def _writeElapsedTime(self, test): + color = get_elapsed_time_color(self.elapsed_time) + self.colorizer.write(" %.2f" % self.elapsed_time, color) + + def _writeResult(self, test, long_result, color, short_result, success): + if self.showAll: + self.colorizer.write(long_result, color) + if self.show_elapsed and success: + self._writeElapsedTime(test) + self.stream.writeln() + elif self.dots: + self.stream.write(short_result) + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + self._handleElapsedTime(test) + self._writeResult(test, 'OK', 'green', '.', True) + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + self._handleElapsedTime(test) + self._writeResult(test, 'FAIL', 'red', 'F', False) + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + self._handleElapsedTime(test) + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + self._writeResult(test, 'ERROR', 'red', 'E', False) + + def startTest(self, test): + unittest.TestResult.startTest(self, test) + self.start_time = time.time() + current_case = test.test.__class__.__name__ + + if self.showAll: + if current_case != self._last_case: + self.stream.writeln(current_case) + self._last_case = current_case + + self.stream.write( + ' %s' % str(test.test._testMethodName).ljust(60)) + self.stream.flush() + + +class NovaTestRunner(core.TextTestRunner): + def __init__(self, *args, **kwargs): + self.show_elapsed = kwargs.pop('show_elapsed') + core.TextTestRunner.__init__(self, *args, **kwargs) + + def _makeResult(self): + return NovaTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config, + show_elapsed=self.show_elapsed) + + def _writeSlowTests(self, result_): + # Pare out 'fast' tests + slow_tests = [item for item in result_.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + self.stream.writeln("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + for elapsed_time, test in sorted(slow_tests, reverse=True): + time_str = "%.2f" % elapsed_time + self.stream.writeln(" %s %s" % (time_str.ljust(10), test)) + + def run(self, test): + result_ = core.TextTestRunner.run(self, test) + if self.show_elapsed: + self._writeSlowTests(result_) + return result_ + + +if __name__ == '__main__': + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + show_elapsed = True + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('tests.%s' % x) + elif x.startswith('--hide-elapsed'): + show_elapsed = False + else: + argv.append(x) + + testdir = os.path.abspath(os.path.join("tests")) + c = config.Config(stream=sys.stdout, + env=os.environ, + verbosity=3, + workingDir=testdir, + plugins=core.DefaultPluginManager()) + + runner = NovaTestRunner(stream=c.stream, + verbosity=c.verbosity, + config=c, + show_elapsed=show_elapsed) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000000..2f6fe3bab5 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -eu + +function usage { + echo "Usage: $0 [OPTION]..." + echo "Run KeystoneLight's test suite(s)" + echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -r, --recreate-db Recreate the test database (deprecated, as this is now the default)." + echo " -n, --no-recreate-db Don't recreate the test database." + echo " -x, --stop Stop running tests after the first error or failure." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." + echo " -p, --pep8 Just run pep8" + echo " -P, --no-pep8 Don't run pep8" + echo " -c, --coverage Generate coverage report" + echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" + echo "" + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." + exit +} + +function process_option { + case "$1" in + -h|--help) usage;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -r|--recreate-db) recreate_db=1;; + -n|--no-recreate-db) recreate_db=0;; + -f|--force) force=1;; + -p|--pep8) just_pep8=1;; + -P|--no-pep8) no_pep8=1;; + -c|--coverage) coverage=1;; + -*) noseopts="$noseopts $1";; + *) noseargs="$noseargs $1" + esac +} + +venv=.ksl-venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +force=0 +noseargs= +noseopts= +wrapper="" +just_pep8=0 +no_pep8=0 +coverage=0 +recreate_db=1 + +for arg in "$@"; do + process_option $arg +done + +# If enabled, tell nose to collect coverage data +if [ $coverage -eq 1 ]; then + noseopts="$noseopts --with-coverage --cover-package=nova" +fi + +function run_tests { + # Just run the test suites in current environment + ${wrapper} $NOSETESTS 2> run_tests.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.log + fi + fi + return $RESULT +} + +function run_pep8 { + echo "Running pep8 ..." + # Opt-out files from pep8 + ignore_scripts="*.sh:" + ignore_files="*eventlet-patch:*pip-requires" + ignore_dirs="*ajaxterm*" + GLOBIGNORE="$ignore_scripts:$ignore_files:$ignore_dirs" + srcfiles=`find bin -type f` + srcfiles+=" keystonelight" + # Just run PEP8 in current environment + ${wrapper} pep8 --repeat --show-pep8 --show-source \ + --ignore=E202,E111 \ + --exclude=vcsversion.py ${srcfiles} +} + +NOSETESTS="python run_tests.py $noseopts $noseargs" + +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py + wrapper=${with_venv} + fi + fi + fi +fi + +# Delete old coverage data from previous runs +if [ $coverage -eq 1 ]; then + ${wrapper} coverage erase +fi + +if [ $just_pep8 -eq 1 ]; then + run_pep8 + exit +fi + +if [ $recreate_db -eq 1 ]; then + rm -f tests.sqlite +fi + +run_tests + +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (noseargs). +if [ -z "$noseargs" ]; then + if [ $no_pep8 -eq 0 ]; then + run_pep8 + fi +fi + +if [ $coverage -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${wrapper} coverage html -d covhtml -i +fi diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000000..c2489f29e1 --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,132 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2011 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +virtualenv installation script +""" + +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.ksl-venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires-test') +PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + + +def die(message, *args): + print >>sys.stderr, message % args + sys.exit(1) + + +def check_python_version(): + if sys.version_info < (2, 6): + die("Need Python Version >= 2.6") + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + """ + Runs a command in an out-of-process shell, returning the + output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return output + + +HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], + check_exit_code=False).strip()) +HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], + check_exit_code=False).strip()) + + +def check_dependencies(): + """Make sure virtualenv is in the path.""" + + print 'Checking for virtualenv...' + if not HAS_VIRTUALENV: + print 'not found.' + # Try installing it via easy_install... + if HAS_EASY_INSTALL: + print 'Installing virtualenv via easy_install...', + if not (run_command(['which', 'easy_install']) and + run_command(['easy_install', 'virtualenv'])): + die('ERROR: virtualenv not found.\n\nNova development' + ' requires virtualenv, please install it using your' + ' favorite package management tool') + print 'done.' + print 'done.' + + +def create_virtualenv(venv=VENV): + """Creates the virtual environment and installs PIP only into the + virtual environment + """ + print 'Creating venv...', + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', 'pip']).strip(): + die("Failed to install pip.") + print 'done.' + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, '-r', + PIP_REQUIRES], redirect_output=False) + + +def print_help(): + help = """ + Virtual environment configuration complete. + + To activate the virtualenv for the extent of your current shell + session you can run: + + $ source %s/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + """ % VENV + print help + + +def main(argv): + check_python_version() + check_dependencies() + create_virtualenv() + install_dependencies() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 769396eb9b..257c0f7e57 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -1,2 +1,14 @@ +# keystonelight dependencies +pam==0.1.4 +WebOb==0.9.8 +eventlet==0.9.12 +PasteDeploy +paste +routes + +# keystonelight testing dependencies +nose + # for python-keystoneclient httplib2 +pep8 diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100755 index 0000000000..0ed2ef7268 --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,4 @@ +#!/bin/bash +TOOLS=`dirname $0` +VENV=$TOOLS/../.ksl-venv +source $VENV/bin/activate && $@ From 3439a776534a90fc0a5ad3414b115ac1aee04c12 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 14:54:50 -0800 Subject: [PATCH 040/334] add novaclient, intermediate --- run_tests.py | 4 +- tests/test_keystoneclient_compat.py | 1 + tests/test_novaclient_compat.py | 75 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/test_novaclient_compat.py diff --git a/run_tests.py b/run_tests.py index 2120373791..98ecb4a9d4 100644 --- a/run_tests.py +++ b/run_tests.py @@ -340,7 +340,9 @@ if __name__ == '__main__': argv = [] for x in sys.argv: if x.startswith('test_'): - argv.append('tests.%s' % x) + pass + #argv.append('tests.%s' % x) + argv.append(x) elif x.startswith('--hide-elapsed'): show_elapsed = False else: diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 1f73adb2fc..30854fbb18 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -70,6 +70,7 @@ class MasterCompatTestCase(CompatTestCase): port = self.server.socket_info['socket'][1] self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, username='foo', password='foo', diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py new file mode 100644 index 0000000000..d14d5ee482 --- /dev/null +++ b/tests/test_novaclient_compat.py @@ -0,0 +1,75 @@ +import copy +import json +import os +import sys + +from keystonelight import logging +from keystonelight import models +from keystonelight import test +from keystonelight import utils + + +KEYSTONECLIENT_REPO = 'git://github.com/rackspace/python-novaclient.git' + + +class CompatTestCase(test.TestCase): + def setUp(self): + super(CompatTestCase, self).setUp() + + +class MasterCompatTestCase(CompatTestCase): + def setUp(self): + super(MasterCompatTestCase, self).setUp() + + revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') + self.add_path(revdir) + from novaclient.keystone import client as ks_client + from novaclient import client as base_client + reload(ks_client) + reload(base_client) + + self.app = self.loadapp('keystoneclient_compat_master') + self.options = self.appconfig('keystoneclient_compat_master') + + self.identity_backend = utils.import_object( + self.options['identity_driver'], options=self.options) + self.token_backend = utils.import_object( + self.options['token_driver'], options=self.options) + self.catalog_backend = utils.import_object( + self.options['catalog_driver'], options=self.options) + + self.server = self.serveapp('keystoneclient_compat_master') + + self.tenant_bar = self.identity_backend._create_tenant( + 'bar', + models.Tenant(id='bar', name='BAR')) + + self.user_foo = self.identity_backend._create_user( + 'foo', + models.User(id='foo', + name='FOO', + tenants=[self.tenant_bar['id']], + password='foo')) + + self.extras_bar_foo = self.identity_backend._create_extras( + self.user_foo['id'], self.tenant_bar['id'], + dict(roles=[], + roles_links=[])) + + + def test_authenticate_and_tenants(self): + from novaclient.keystone import client as ks_client + from novaclient import client as base_client + + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, + user='foo', + apikey='foo', + projectid='bar') + client = ks_client.Client(conn) + client.authenticate() + #tenants = client.tenants.list() + #self.assertEquals(tenants[0].id, self.tenant_bar['id']) From 44a07fd887e6ae46a5fca2958dc776979338c2c6 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 7 Nov 2011 19:26:39 -0800 Subject: [PATCH 041/334] get novaclient tests working --- tests/keystoneclient_compat_master.conf | 9 ++++++--- tests/test_novaclient_compat.py | 17 +++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index 049759171a..091071c779 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -11,10 +11,13 @@ catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 catalog.RegionOne.identity.name = 'Identity Service' +# fake compute port for now to help novaclient tests work +compute_port = 3000 +catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v2.0 +catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v2.0 +catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v2.0 +catalog.RegionOne.compute.name = 'Compute Service' -# in old config: -# first 1 == public -# second 1 == enabled [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index d14d5ee482..73a9f533ad 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -17,9 +17,9 @@ class CompatTestCase(test.TestCase): super(CompatTestCase, self).setUp() -class MasterCompatTestCase(CompatTestCase): +class NovaClientCompatMasterTestCase(CompatTestCase): def setUp(self): - super(MasterCompatTestCase, self).setUp() + super(NovaClientCompatMasterTestCase, self).setUp() revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') self.add_path(revdir) @@ -64,12 +64,17 @@ class MasterCompatTestCase(CompatTestCase): port = self.server.socket_info['socket'][1] self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + # NOTE(termie): novaclient wants a "/" TypeErrorat the end, keystoneclient does not + # NOTE(termie): projectid is apparently sent as tenantName, so... that's + # unfortunate. + # NOTE(termie): novaclient seems to care about the region more than + # keystoneclient conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, user='foo', apikey='foo', - projectid='bar') + projectid='BAR', + region_name='RegionOne') client = ks_client.Client(conn) client.authenticate() - #tenants = client.tenants.list() - #self.assertEquals(tenants[0].id, self.tenant_bar['id']) + # NOTE(termie): novaclient doesn't know about tenants or anything like that + # so just test that we can validate From 32121019dcf5448070942752f633eab3db5559ff Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 4 Nov 2011 15:56:17 -0700 Subject: [PATCH 042/334] add crud info to readme --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index 3277d44468..600aaa3ce5 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,9 @@ Keystone Light is a re-interpretation of the Keystone project that provides Identity, Token and Catalog services for use specifically by projects in the OpenStack family. +Much of the design is precipitated from the expectation that the auth backends +for most deployments will actually be shims in front of existing user systems. + ---------- Data Model @@ -87,6 +90,18 @@ Diagram: keystone_compat_flows.sdx_ .. _: https://raw.github.com/termie/keystonelight/master/docs/keystone_compat_flows.sdx +---------------- +Approach to CRUD +---------------- + +While it is expected that any "real" deployment at a large company will manage +their users, tenants and other metadata in their existing user systems, a +variety of CRUD operations are provided for the sake of development and testing. + +CRUD is treated as an extension or additional feature to the core feature set in +that it is not required that a backend support it. + + ----------- Still To Do ----------- From 8fd822002ea52c156bcb04319b36c9fe5272c816 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 8 Nov 2011 13:04:26 -0800 Subject: [PATCH 043/334] update keystone sample tests, skip one --- tests/test_keystone_compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index 609d3a6427..ab5e3c7291 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -3,6 +3,8 @@ import json import os import sys +from nose import exc + from keystonelight import logging from keystonelight import models from keystonelight import test @@ -125,6 +127,9 @@ class DiabloCompatTestCase(CompatTestCase): super(DiabloCompatTestCase, self).setUp() def test_authenticate_scoped(self): + # NOTE(termie): the docs arbitrarily changed and inserted a 'u' in front + # of one of the user ids, but none of the others + raise exc.SkipTest() client = self.client(self.app) post_data = json.dumps( {'auth': {'passwordCredentials': {'username': self.user_123['id'], From 3d79099bace07244a0332fc8e2b1e74a548f6049 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 8 Nov 2011 13:28:40 -0800 Subject: [PATCH 044/334] add a trivial admin-only middleware --- README.rst | 2 +- keystonelight/keystone_compat.py | 4 ++++ keystonelight/service.py | 14 ++++++++++++++ tests/default.conf | 6 +++++- tests/keystone_compat_diablo.conf | 6 +++++- tests/keystoneclient_compat_master.conf | 6 +++++- tests/test_keystone_compat.py | 11 +++-------- 7 files changed, 37 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 600aaa3ce5..c0faefecbf 100644 --- a/README.rst +++ b/README.rst @@ -111,5 +111,5 @@ Still To Do * Fixture loading functionality would also be killer tests and dev. * LDAP backend. * Keystone import. - * Admin-only interface + * (./) Admin-only interface * Don't check git checkouts as often, to speed up tests diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index cf99244c04..0bb0ed4144 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -186,6 +186,8 @@ class KeystoneController(service.BaseApplication): Optionally, also ensure that it is owned by a specific tenant. """ + assert context['is_admin'] + token_ref = self.token_api.get_token(context=context, token_id=token_id) if belongs_to: @@ -220,6 +222,8 @@ class KeystoneController(service.BaseApplication): """ token_ref = self.token_api.get_token(context=context, token_id=context['token_id']) + assert token_ref is not None + user_ref = token_ref['user'] tenant_refs = [] for tenant_id in user_ref['tenants']: diff --git a/keystonelight/service.py b/keystonelight/service.py index 799bd354af..797a341557 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -52,6 +52,20 @@ class TokenAuthMiddleware(wsgi.Middleware): request.environ['openstack.context'] = context +class AdminTokenAuthMiddleware(wsgi.Middleware): + """A trivial filter that checks for a pre-defined admin token. + + Sets 'is_admin' to true in the context, expected to be checked by + methods that are admin-only. + + """ + def process_request(self, request): + token = request.headers.get('X-Auth-Token') + context = request.environ.get('openstack.context', {}) + context['is_admin'] = (token == self.options['admin_token']) + request.environ['openstack.context'] = context + + class PostParamsMiddleware(wsgi.Middleware): """Middleware to allow method arguments to be passed as POST parameters. diff --git a/tests/default.conf b/tests/default.conf index fa6ac28b52..68388b27d1 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -2,6 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +admin_token = ADMIN [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory @@ -9,6 +10,9 @@ paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +[filter:admin_token_auth] +paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory + [filter:json_body] paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory @@ -16,4 +20,4 @@ paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory paste.app_factory = keystonelight.service:app_factory [pipeline:main] -pipeline = token_auth json_body debug keystonelight +pipeline = token_auth admin_token_auth json_body debug keystonelight diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index 8318a0ca7c..d90526316e 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -3,6 +3,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken public_port = 5000 +admin_token = ADMIN [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory @@ -10,6 +11,9 @@ paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +[filter:admin_token_auth] +paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory + [filter:json_body] paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory @@ -17,4 +21,4 @@ paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory paste.app_factory = keystonelight.keystone_compat:app_factory [pipeline:main] -pipeline = token_auth json_body debug keystone +pipeline = token_auth admin_token_auth json_body debug keystone diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index 091071c779..e006e82101 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -3,6 +3,7 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken public_port = 5000 +admin_token = ADMIN # config for TemplatedCatalog, using camelCase because I don't want to do # translations for keystone compat @@ -25,6 +26,9 @@ paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +[filter:admin_token_auth] +paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory + [filter:json_body] paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory @@ -32,4 +36,4 @@ paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory paste.app_factory = keystonelight.keystone_compat:app_factory [pipeline:main] -pipeline = token_auth json_body debug keystone +pipeline = token_auth admin_token_auth json_body debug keystone diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index ab5e3c7291..a32477ca26 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -40,6 +40,7 @@ class CompatTestCase(test.TestCase): # NOTE(termie): stupid hack to deal with the keystone samples being # completely inconsistent self.validate_token['access']['user']['roles'][1]['id'] = u'235' + self.admin_token = 'ADMIN' self.auth_response = json.load(open( os.path.join(self.sampledir, 'auth.json'))) @@ -129,7 +130,7 @@ class DiabloCompatTestCase(CompatTestCase): def test_authenticate_scoped(self): # NOTE(termie): the docs arbitrarily changed and inserted a 'u' in front # of one of the user ids, but none of the others - raise exc.SkipTest() + raise exc.SkipTest('The docs have arbitrarily changed.') client = self.client(self.app) post_data = json.dumps( {'auth': {'passwordCredentials': {'username': self.user_123['id'], @@ -149,13 +150,7 @@ class DiabloCompatTestCase(CompatTestCase): # data['access']['serviceCatalog']) def test_validate_token_scoped(self): - client = self.client(self.app, token=self.token_123['id']) - resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) - data = json.loads(resp.body) - self.assertDeepEquals(self.validate_token, data) - - def test_validate_token_scoped(self): - client = self.client(self.app, token=self.token_123['id']) + client = self.client(self.app, token=self.admin_token) resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) data = json.loads(resp.body) self.assertDeepEquals(self.validate_token, data) From 3117b4188649556fe9f6487b667e20a0acbc9f7e Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 8 Nov 2011 13:43:03 -0800 Subject: [PATCH 045/334] use paste for the binary --- bin/keystone | 29 +++++++---------------------- keystonelight/logging.py | 4 ++-- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/bin/keystone b/bin/keystone index 3f3620fa62..b0e31f04ae 100755 --- a/bin/keystone +++ b/bin/keystone @@ -14,33 +14,18 @@ if os.path.exists(os.path.join(possible_topdir, '__init__.py')): sys.path.insert(0, possible_topdir) -import logging +from paste import deploy -import hflags as flags - -from keystonelight import service from keystonelight import wsgi -FLAGS = flags.FLAGS - -flags.DEFINE_boolean('verbose', True, 'verbose logging') -flags.DEFINE_flag(flags.HelpFlag()) - if __name__ == '__main__': - args = FLAGS(sys.argv) - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - - public = service.Router() - admin = service.AdminRouter() - - public = service.TokenAuthMiddleware(public) - public = service.JsonBodyMiddleware(public) - - admin = service.TokenAuthMiddleware(admin) + default_conf = os.path.join(possible_topdir, + 'etc', + 'keystone.conf') + conf = len(sys.argv) > 1 and sys.argv[1] or default_conf + app = deploy.loadapp('config:%s' % conf) server = wsgi.Server() - server.start(public, 8080) - server.start(admin, 8081) + server.start(app, int(app.options['public_port'])) server.wait() diff --git a/keystonelight/logging.py b/keystonelight/logging.py index 1de4bdadc2..c1543961e2 100644 --- a/keystonelight/logging.py +++ b/keystonelight/logging.py @@ -31,9 +31,9 @@ log = logging.log # handlers StreamHandler = logging.StreamHandler -WatchedFileHandler = logging.handlers.WatchedFileHandler +#WatchedFileHandler = logging.handlers.WatchedFileHandler # logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. -SysLogHandler = logging.handlers.SysLogHandler +#SysLogHandler = logging.handlers.SysLogHandler def log_debug(f): From 8ae627a317f91797fc3331f8f8eda0e7682ecc36 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 8 Nov 2011 15:03:15 -0800 Subject: [PATCH 046/334] add a stubby setup.py --- bin/ksl | 0 keystonelight/backends/kvs.py | 13 +++++++++++++ setup.py | 13 +++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 bin/ksl create mode 100755 setup.py diff --git a/bin/ksl b/bin/ksl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 09baa75922..e7fa369fbb 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -52,6 +52,19 @@ class KvsIdentity(object): def get_extras(self, user_id, tenant_id): return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + def create_user(self, id, user): + self.db.set('user-%s' % id, user) + return user + + def create_tenant(self, id, tenant): + self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def create_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + # Private CRUD for testing def _create_user(self, id, user): self.db.set('user-%s' % id, user) diff --git a/setup.py b/setup.py new file mode 100755 index 0000000000..21099e48e1 --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup(name='keystonelight', + version='1.0', + description="Authentication service for OpenStack", + author='OpenStack, LLC.', + author_email='openstack@lists.launchpad.net', + url='http://www.openstack.org', + packages=find_packages(exclude=['test', 'bin']), + scripts=['bin/keystone', 'bin/ksl'], + zip_safe=False, + install_requires=['setuptools'], + ) From cd712b2643b818ffb4272c4b3b546685c48e9640 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 8 Nov 2011 16:05:48 -0800 Subject: [PATCH 047/334] add a default handler for / --- bin/keystone | 2 ++ keystonelight/keystone_compat.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/bin/keystone b/bin/keystone index b0e31f04ae..75663d1b5e 100755 --- a/bin/keystone +++ b/bin/keystone @@ -1,6 +1,7 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import logging import os import sys @@ -23,6 +24,7 @@ if __name__ == '__main__': default_conf = os.path.join(possible_topdir, 'etc', 'keystone.conf') + logging.getLogger().setLevel(logging.DEBUG) conf = len(sys.argv) > 1 and sys.argv[1] or default_conf app = deploy.loadapp('config:%s' % conf) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 0bb0ed4144..f751a905ef 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -18,6 +18,9 @@ class KeystoneRouter(wsgi.Router): self.keystone_controller = KeystoneController(options) mapper = routes.Mapper() + mapper.connect('/', + controller=self.keystone_controller, + action='noop') mapper.connect('/v2.0/tokens', controller=self.keystone_controller, action='authenticate', @@ -41,6 +44,9 @@ class KeystoneController(service.BaseApplication): self.token_api = token.Manager(options) pass + def noop(self, context): + return {} + def authenticate(self, context, auth=None): """Authenticate credentials and return a token. From 4885d4a2d3673c6a892cb6724797544be62cca69 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 9 Nov 2011 11:57:59 -0800 Subject: [PATCH 048/334] add an etc dir --- etc/keystone.conf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 etc/keystone.conf diff --git a/etc/keystone.conf b/etc/keystone.conf new file mode 100644 index 0000000000..d90526316e --- /dev/null +++ b/etc/keystone.conf @@ -0,0 +1,24 @@ +[DEFAULT] +catalog_driver = keystonelight.backends.kvs.KvsCatalog +identity_driver = keystonelight.backends.kvs.KvsIdentity +token_driver = keystonelight.backends.kvs.KvsToken +public_port = 5000 +admin_token = ADMIN + +[filter:debug] +paste.filter_factory = keystonelight.wsgi:Debug.factory + +[filter:token_auth] +paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory + +[filter:admin_token_auth] +paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory + +[filter:json_body] +paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory + +[app:keystone] +paste.app_factory = keystonelight.keystone_compat:app_factory + +[pipeline:main] +pipeline = token_auth admin_token_auth json_body debug keystone From 64b369f4d34ec2aafd383dfc15b4293db6b7a14e Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 9 Nov 2011 12:37:58 -0800 Subject: [PATCH 049/334] add admin port --- bin/keystone | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/keystone b/bin/keystone index 75663d1b5e..f353e8f9b6 100755 --- a/bin/keystone +++ b/bin/keystone @@ -30,4 +30,5 @@ if __name__ == '__main__': app = deploy.loadapp('config:%s' % conf) server = wsgi.Server() server.start(app, int(app.options['public_port'])) + server.start(app, int(app.options['admin_port'])) server.wait() From 570b08d189dd33d74afa45dc0270ffc6ba2030b8 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 9 Nov 2011 14:38:34 -0800 Subject: [PATCH 050/334] cli beginnings --- bin/ksl | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) mode change 100644 => 100755 bin/ksl diff --git a/bin/ksl b/bin/ksl old mode 100644 new mode 100755 index e69de29bb2..2005ee4407 --- a/bin/ksl +++ b/bin/ksl @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +import sys + +import cli.app +import cli.log + + +DEFAULT_PARAMS = ( + (('--url',), {'dest': 'url', 'action': 'store'}), + (('--token',), {'dest': 'token', 'action': 'store'}), + ) + + +class BaseApp(cli.log.LoggingApp): + def __init__(self, *args, **kw): + kw.setdefault('name', self.__class__.__name__.lower()) + super(BaseApp, self).__init__(*args, **kw) + + def add_default_params(self): + for args, kw in DEFAULT_PARAMS: + self.add_param(*args, **kw) + + +class LoadData(BaseApp): + def __init__(self, *args, **kw): + super(LoadData, self).__init__(*args, **kw) + self.add_default_params() + self.add_param('fixture', nargs='+') + + def main(self): + """Given some fixtures, create the appropriate data in Keystone Light.""" + pass + + + +CMDS = {'loaddata': LoadData, + } + + +if __name__ == '__main__': + if not len(sys.argv) > 1: + print 'try one of:', ' '.join(CMDS.keys()) + sys.exit(1) + + cmd = sys.argv[1] + if cmd in CMDS: + CMDS[cmd](argv=(sys.argv[:1] + sys.argv[2:])).run() From 59c2dea30f0fc4db438c4515883f2667c989939c Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 11:03:40 -0800 Subject: [PATCH 051/334] add crud methods to identity manager --- keystonelight/identity.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index a144e938f5..1b2a2f30ac 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -32,3 +32,31 @@ class Manager(object): def get_extras(self, context, user_id, tenant_id): return self.driver.get_extras(user_id, tenant_id) + + # CRUD operations + def create_user(self, context, user_id, data): + return self.driver.create_user(user_id, data) + + def update_user(self, context, user_id, data): + return self.driver.update_user(user_id, data) + + def delete_user(self, context, user_id): + return self.driver.delete_user(user_id) + + def create_tenant(self, context, tenant_id, data): + return self.driver.create_tenant(tenant_id, data) + + def update_tenant(self, context, tenant_id, data): + return self.driver.update_tenant(tenant_id, data) + + def delete_tenant(self, context, tenant_id): + return self.driver.delete_tenant(tenant_id) + + def create_extras(self, context, user_id, tenant_id, data): + return self.driver.create_extras(user_id, tenant_id, data) + + def update_extras(self, context, user_id, tenant_id, data): + return self.driver.update_extras(user_id, tenant_id, data) + + def delete_extras(self, context, user_id, tenant_id): + return self.driver.delete_extras(user_id, tenant_id) From 716c450fbfb372245c7fb4b24567df7782a4b28b Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 11:03:54 -0800 Subject: [PATCH 052/334] make a composite app --- tests/default.conf | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/default.conf b/tests/default.conf index 68388b27d1..39ad034ef1 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -1,9 +1,25 @@ [DEFAULT] -catalog_driver = keystonelight.backends.kvs.KvsCatalog +catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +public_port = 5000 admin_token = ADMIN +# config for TemplatedCatalog, using camelCase because I don't want to do +# translations for keystone compat +catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.name = 'Identity Service' + +# fake compute service for now to help novaclient tests work +compute_port = 3000 +catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.name = 'Compute Service' + + [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory @@ -19,5 +35,16 @@ paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[pipeline:main] +[app:keystone] +paste.app_factory = keystonelight.keystone_compat:app_factory + +[pipeline:keystone_api] +pipeline = token_auth admin_token_auth json_body debug keystone + +[pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight + +[composite:main] +use = egg:Paste#urlmap +/ = keystonelight_api +/v2.0 = keystone_api From d7f364e2098a10a8922996618acc11a8c341a117 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 11:17:07 -0800 Subject: [PATCH 053/334] move around middleware --- keystonelight/middleware.py | 96 +++++++++++++++++++++ keystonelight/service.py | 126 +++------------------------ tests/default.conf | 6 +- tests/test_identity_api.py | 166 ++++++++++++++++++++++++++++++++++++ 4 files changed, 275 insertions(+), 119 deletions(-) create mode 100644 keystonelight/middleware.py create mode 100644 tests/test_identity_api.py diff --git a/keystonelight/middleware.py b/keystonelight/middleware.py new file mode 100644 index 0000000000..f32d59c6b1 --- /dev/null +++ b/keystonelight/middleware.py @@ -0,0 +1,96 @@ +import json + +from keystonelight import wsgi + + +# Header used to transmit the auth token +AUTH_TOKEN_HEADER = 'X-Auth-Token' + + +# Environment variable used to pass the request context +CONTEXT_ENV = 'openstack.context' + + +# Environment variable used to pass the request params +PARAMS_ENV = 'openstack.params' + + +class TokenAuthMiddleware(wsgi.Middleware): + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['token_id'] = token + request.environ[CONTEXT_ENV] = context + + +class AdminTokenAuthMiddleware(wsgi.Middleware): + """A trivial filter that checks for a pre-defined admin token. + + Sets 'is_admin' to true in the context, expected to be checked by + methods that are admin-only. + + """ + + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['is_admin'] = (token == self.options['admin_token']) + request.environ[CONTEXT_ENV] = context + + +class PostParamsMiddleware(wsgi.Middleware): + """Middleware to allow method arguments to be passed as POST parameters. + + Filters out the parameters `self`, `context` and anything beginning with + an underscore. + + """ + + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ[PARAMS_ENV] = params + + +class JsonBodyMiddleware(wsgi.Middleware): + """Middleware to allow method arguments to be passed as serialized JSON. + + Accepting arguments as JSON is useful for accepting data that may be more + complex than simple primitives. + + In this case we accept it as urlencoded data under the key 'json' as in + json= but this could be extended to accept raw JSON + in the POST body. + + Filters out the parameters `self`, `context` and anything beginning with + an underscore. + + """ + + def process_request(self, request): + #if 'json' not in request.params: + # return + + params_json = request.body + if not params_json: + return + + params_parsed = json.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ[PARAMS_ENV] = params + + diff --git a/keystonelight/service.py b/keystonelight/service.py index 797a341557..e4e3e9a0d2 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -44,84 +44,6 @@ class BaseApplication(wsgi.Application): return json.dumps(result) -class TokenAuthMiddleware(wsgi.Middleware): - def process_request(self, request): - token = request.headers.get('X-Auth-Token') - context = request.environ.get('openstack.context', {}) - context['token_id'] = token - request.environ['openstack.context'] = context - - -class AdminTokenAuthMiddleware(wsgi.Middleware): - """A trivial filter that checks for a pre-defined admin token. - - Sets 'is_admin' to true in the context, expected to be checked by - methods that are admin-only. - - """ - def process_request(self, request): - token = request.headers.get('X-Auth-Token') - context = request.environ.get('openstack.context', {}) - context['is_admin'] = (token == self.options['admin_token']) - request.environ['openstack.context'] = context - - -class PostParamsMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as POST parameters. - - Filters out the parameters `self`, `context` and anything beginning with - an underscore. - - """ - - def process_request(self, request): - params_parsed = request.params - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v - - request.environ['openstack.params'] = params - - -class JsonBodyMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as serialized JSON. - - Accepting arguments as JSON is useful for accepting data that may be more - complex than simple primitives. - - In this case we accept it as urlencoded data under the key 'json' as in - json= but this could be extended to accept raw JSON - in the POST body. - - Filters out the parameters `self`, `context` and anything beginning with - an underscore. - - """ - - def process_request(self, request): - #if 'json' not in request.params: - # return - - params_json = request.body - if not params_json: - return - - params_parsed = json.loads(params_json) - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v - - request.environ['openstack.params'] = params - - class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" @@ -169,51 +91,23 @@ class IdentityController(BaseApplication): class Router(wsgi.Router): def __init__(self, options): self.options = options - token_controller = utils.import_object( - options['token_controller'], - options=options) - identity_controller = utils.import_object( - options['identity_controller'], - options=options) + self.identity_controller = IdentityController(options) + self.token_controller = TokenController(options) mapper = routes.Mapper() - mapper.connect('/v2.0/tokens', controller=identity_controller, + mapper.connect('/v2.0/tokens', + controller=self.identity_controller, action='authenticate') - mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, + mapper.connect('/v2.0/tokens/{token_id}', + controller=self.token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) - mapper.connect("/v2.0/tenants", controller=identity_controller, - action="get_tenants", conditions=dict(method=["GET"])) + mapper.connect("/v2.0/tenants", + controller=self.identity_controller, + action="get_tenants", + conditions=dict(method=["GET"])) super(Router, self).__init__(mapper) -class AdminRouter(wsgi.Router): - def __init__(self, options): - self.options = options - token_controller = utils.import_object( - options['token_controller'], - options=options) - identity_controller = utils.import_object( - options['identity_controller'], - options=options) - mapper = routes.Mapper() - - mapper.connect('/v2.0/tokens', controller=identity_controller, - action='authenticate') - mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, - action='validate_token', - conditions=dict(method=['GET'])) - mapper.connect('/v2.0/tokens/{token_id}', controller=token_controller, - action='revoke_token', - conditions=dict(method=['DELETE'])) - super(AdminRouter, self).__init__(mapper) - - -def identity_app_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - return Router(conf) - - def app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) diff --git a/tests/default.conf b/tests/default.conf index 39ad034ef1..220a350f9e 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -24,13 +24,13 @@ catalog.RegionOne.compute.name = 'Compute Service' paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory +paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py new file mode 100644 index 0000000000..ed1135efac --- /dev/null +++ b/tests/test_identity_api.py @@ -0,0 +1,166 @@ +import uuid + +from keystonelight import models +from keystonelight import test +from keystonelight import utils +from keystonelight.backends import kvs + + +class IdentityApi(test.TestCase): + def setUp(self): + super(IdentityApi, self).setUp() + self.options = self.appconfig('default') + app = self.loadapp('default') + self.app = app + + self.identity_backend = utils.import_object( + self.options['identity_driver'], options=self.options) + self.token_backend = utils.import_object( + self.options['token_driver'], options=self.options) + self.catalog_backend = utils.import_object( + self.options['catalog_driver'], options=self.options) + self._load_fixtures() + + def _load_fixtures(self): + self.tenant_bar = self.identity_backend._create_tenant( + 'bar', + models.Tenant(id='bar', name='BAR')) + self.user_foo = self.identity_backend._create_user( + 'foo', + models.User(id='foo', + name='FOO', + password='foo2', + tenants=[self.tenant_bar['id']])) + self.extras_foobar = self.identity_backend._create_extras( + 'foo', 'bar', + {'extra': 'extra'}) + + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') + + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) + + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(extras_ref is None) + + def test_authenticate(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) + + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + self.assertDictEquals(user_ref, self.user_foo) + + def test_get_extras_bad_user(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(extras_ref is None) + + def test_get_extras_bad_tenant(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(extras_ref is None) + + def test_get_extras(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(extras_ref, self.extras_foobar) + + +class KvsToken(test.TestCase): + def setUp(self): + super(KvsToken, self).setUp() + options = self.appconfig('default') + self.token_api = kvs.KvsToken(options=options, db={}) + + def test_token_crud(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, + 'a': 'b'} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + + new_data_ref = self.token_api.get_token(token_id) + self.assertEquals(new_data_ref, data) + + self.token_api.delete_token(token_id) + deleted_data_ref = self.token_api.get_token(token_id) + self.assert_(deleted_data_ref is None) + + +class KvsCatalog(test.TestCase): + def setUp(self): + super(KvsCatalog, self).setUp() + options = self.appconfig('default') + self.catalog_api = kvs.KvsCatalog(options=options, db={}) + self._load_fixtures() + + def _load_fixtures(self): + self.catalog_foobar = self.catalog_api._create_catalog( + 'foo', 'bar', + {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) + + def test_get_catalog_bad_user(self): + catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') + self.assert_(catalog_ref is None) + + def test_get_catalog_bad_tenant(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') + self.assert_(catalog_ref is None) + + def test_get_catalog(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar') + self.assertDictEquals(catalog_ref, self.catalog_foobar) From 4b4969f7dc9f895cf03286fbcd6ff006503b6d71 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 14:12:25 -0800 Subject: [PATCH 054/334] update service to middleware in confs --- tests/keystone_compat_diablo.conf | 6 +++--- tests/keystoneclient_compat_master.conf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index d90526316e..94024df33f 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -9,13 +9,13 @@ admin_token = ADMIN paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory +paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index e006e82101..e9861b6b98 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -24,13 +24,13 @@ catalog.RegionOne.compute.name = 'Compute Service' paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory +paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory From 84644998b3d4b6321bd27ca21ded89b371ef1957 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 14:57:42 -0800 Subject: [PATCH 055/334] get some initial identity api tests working --- bin/ksl | 1 - keystonelight/client.py | 67 +++++++++++++++ keystonelight/middleware.py | 2 - keystonelight/service.py | 35 ++++---- tests/test_identity_api.py | 159 ++++++++---------------------------- 5 files changed, 121 insertions(+), 143 deletions(-) create mode 100644 keystonelight/client.py diff --git a/bin/ksl b/bin/ksl index 2005ee4407..e8f2587a4a 100755 --- a/bin/ksl +++ b/bin/ksl @@ -33,7 +33,6 @@ class LoadData(BaseApp): pass - CMDS = {'loaddata': LoadData, } diff --git a/keystonelight/client.py b/keystonelight/client.py new file mode 100644 index 0000000000..7b97158489 --- /dev/null +++ b/keystonelight/client.py @@ -0,0 +1,67 @@ + +"""Client library for KeystoneLight API.""" + +import json + +import httplib2 +import webob + +from keystonelight import wsgi + + +class Client(object): + def __init__(self, token=None): + self.token = token + + def request(self, method, path, headers, body): + raise NotImplemented + + def get(self, path, headers=None): + return self.request('GET', path=path, headers=headers) + + def post(self, path, headers=None, body=None): + return self.request('POST', path=path, headers=headers, body=body) + + def put(self, path, headers=None, body=None): + return self.request('PUT', path=path, headers=headers, body=body) + + def _build_headers(self, headers=None): + if headers is None: + headers = {} + + if self.token: + headers.setdefault('X-Auth-Token', self.token) + + return headers + + +class HttpClient(Client): + def __init__(self, endpoint=None, token=None): + self.endpoint = endpoint + super(HttpClient, self).__init__(token=token) + + def request(self, method, path, headers=None, body=None): + if type(body) is type({}): + body = json.dumps(body) + headers = self._build_headers(headers) + h = httplib.Http() + resp, content = h.request(path, method=method, headers=headers, body=body) + return webob.Response(content, status=resp.status, headerlist=resp.headers) + + +class TestClient(Client): + def __init__(self, app=None, token=None): + self.app = app + super(TestClient, self).__init__(token=token) + + def request(self, method, path, headers=None, body=None): + if type(body) is type({}): + body = json.dumps(body) + headers = self._build_headers(headers) + req = wsgi.Request.blank(path) + req.method = method + for k, v in headers.iteritems(): + req.headers[k] = v + if body: + req.body = body + return req.get_response(self.app) diff --git a/keystonelight/middleware.py b/keystonelight/middleware.py index f32d59c6b1..29e655bd9f 100644 --- a/keystonelight/middleware.py +++ b/keystonelight/middleware.py @@ -92,5 +92,3 @@ class JsonBodyMiddleware(wsgi.Middleware): params[k] = v request.environ[PARAMS_ENV] = params - - diff --git a/keystonelight/service.py b/keystonelight/service.py index e4e3e9a0d2..a4c46e9e04 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -71,21 +71,26 @@ class IdentityController(BaseApplication): self.options = options def authenticate(self, context, **kwargs): - tenant, user, extras = self.identity_api.authenticate(context, - **kwargs) - token = self.token_api.create_token(context, - dict(tenant=tenant, - user=user, - extras=extras)) - logging.debug('TOKEN: %s', token) - return token + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + context, **kwargs) + # TODO(termie): strip password from return values + token_ref = self.token_api.create_token(context, + dict(tenant=tenant_ref, + user=user_ref, + extras=extras_ref)) + logging.debug('TOKEN: %s', token_ref) + return token_ref def get_tenants(self, context): - token_id = context.get('token') - token = self.token_api.validate_token(context, token_id) + token_id = context.get('token_id') + token_ref = self.token_api.get_token(context, token_id) + assert token_ref + tenants_ref = [] + for tenant_id in token_ref['user']['tenants']: + tenants_ref.append(self.identity_api.get_tenant(context, + tenant_id)) - return self.identity_api.get_tenants(context, - user_id=token['user']) + return tenants_ref class Router(wsgi.Router): @@ -94,14 +99,14 @@ class Router(wsgi.Router): self.identity_controller = IdentityController(options) self.token_controller = TokenController(options) mapper = routes.Mapper() - mapper.connect('/v2.0/tokens', + mapper.connect('/tokens', controller=self.identity_controller, action='authenticate') - mapper.connect('/v2.0/tokens/{token_id}', + mapper.connect('/tokens/{token_id}', controller=self.token_controller, action='revoke_token', conditions=dict(method=['DELETE'])) - mapper.connect("/v2.0/tenants", + mapper.connect("/tenants", controller=self.identity_controller, action="get_tenants", conditions=dict(method=["GET"])) diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index ed1135efac..21b2578138 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -1,5 +1,7 @@ +import json import uuid +from keystonelight import client from keystonelight import models from keystonelight import test from keystonelight import utils @@ -35,132 +37,39 @@ class IdentityApi(test.TestCase): 'foo', 'bar', {'extra': 'extra'}) - def test_authenticate_bad_user(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - - def test_authenticate_bad_password(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password'] + 'WRONG') - - def test_authenticate_invalid_tenant(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG', - password=self.user_foo['password']) - - def test_authenticate_no_tenant(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assert_(tenant_ref is None) - self.assert_(extras_ref is None) + def _login(self): + c = client.TestClient(self.app) + post_data = {'user_id': self.user_foo['id'], + 'tenant_id': self.tenant_bar['id'], + 'password': self.user_foo['password']} + resp = c.post('/tokens', body=post_data) + token = json.loads(resp.body) + return token def test_authenticate(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assertDictEquals(tenant_ref, self.tenant_bar) - self.assertDictEquals(extras_ref, self.extras_foobar) + c = client.TestClient(self.app) + post_data = {'user_id': self.user_foo['id'], + 'tenant_id': self.tenant_bar['id'], + 'password': self.user_foo['password']} + resp = c.post('/tokens', body=post_data) + data = json.loads(resp.body) + self.assertEquals(self.user_foo['id'], data['user']['id']) + self.assertEquals(self.tenant_bar['id'], data['tenant']['id']) + self.assertDictEquals(self.extras_foobar, data['extras']) - def test_get_tenant_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(tenant_ref is None) + def test_authenticate_no_tenant(self): + c = client.TestClient(self.app) + post_data = {'user_id': self.user_foo['id'], + 'password': self.user_foo['password']} + resp = c.post('/tokens', body=post_data) + data = json.loads(resp.body) + self.assertEquals(self.user_foo['id'], data['user']['id']) + self.assertEquals(None, data['tenant']) + self.assertEquals(None, data['extras']) - def test_get_tenant(self): - tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_tenant_by_name_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['name'] + 'WRONG') - self.assert_(tenant_ref is None) - - def test_get_tenant_by_name(self): - tenant_ref = self.identity_api.get_tenant_by_name( - tenant_name=self.tenant_bar['name']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_user_bad_user(self): - user_ref = self.identity_api.get_user( - user_id=self.user_foo['id'] + 'WRONG') - self.assert_(user_ref is None) - - def test_get_user(self): - user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) - self.assertDictEquals(user_ref, self.user_foo) - - def test_get_extras_bad_user(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id']) - self.assert_(extras_ref is None) - - def test_get_extras_bad_tenant(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(extras_ref is None) - - def test_get_extras(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id']) - self.assertDictEquals(extras_ref, self.extras_foobar) - - -class KvsToken(test.TestCase): - def setUp(self): - super(KvsToken, self).setUp() - options = self.appconfig('default') - self.token_api = kvs.KvsToken(options=options, db={}) - - def test_token_crud(self): - token_id = uuid.uuid4().hex - data = {'id': token_id, - 'a': 'b'} - data_ref = self.token_api.create_token(token_id, data) - self.assertDictEquals(data_ref, data) - - new_data_ref = self.token_api.get_token(token_id) - self.assertEquals(new_data_ref, data) - - self.token_api.delete_token(token_id) - deleted_data_ref = self.token_api.get_token(token_id) - self.assert_(deleted_data_ref is None) - - -class KvsCatalog(test.TestCase): - def setUp(self): - super(KvsCatalog, self).setUp() - options = self.appconfig('default') - self.catalog_api = kvs.KvsCatalog(options=options, db={}) - self._load_fixtures() - - def _load_fixtures(self): - self.catalog_foobar = self.catalog_api._create_catalog( - 'foo', 'bar', - {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) - - def test_get_catalog_bad_user(self): - catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') - self.assert_(catalog_ref is None) - - def test_get_catalog_bad_tenant(self): - catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') - self.assert_(catalog_ref is None) - - def test_get_catalog(self): - catalog_ref = self.catalog_api.get_catalog('foo', 'bar') - self.assertDictEquals(catalog_ref, self.catalog_foobar) + def test_get_tenants(self): + token = self._login() + c = client.TestClient(self.app, token['id']) + resp = c.get('/tenants') + data = json.loads(resp.body) + self.assertDictEquals(self.tenant_bar, data[0]) From e10512b2b879e4e70a59722d00dcb21f6a940dff Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 10 Nov 2011 15:57:54 -0800 Subject: [PATCH 056/334] more dyanmic client --- keystonelight/client.py | 33 ++++++++++++++++ keystonelight/service.py | 77 +++++++++++++++++++++++++++++--------- tests/test_identity_api.py | 6 +-- 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/keystonelight/client.py b/keystonelight/client.py index 7b97158489..25128ba33f 100644 --- a/keystonelight/client.py +++ b/keystonelight/client.py @@ -6,9 +6,13 @@ import json import httplib2 import webob +from keystonelight import service from keystonelight import wsgi +URLMAP = service.URLMAP + + class Client(object): def __init__(self, token=None): self.token = token @@ -34,6 +38,35 @@ class Client(object): return headers + def __getattr__(self, key): + """Lazy way to define a bunch of dynamic urls based on URLMAP. + + Turns something like + + c.authenticate(user_id='foo', password='bar') + + into + + c.request('POST', '/token', body={'user_id': 'foo', 'password': 'bar'}) + + """ + if key not in URLMAP: + raise AttributeError(key) + + method, path = URLMAP[key] + + def _internal(method_=method, path_=path, **kw): + path_ = path_ % kw + params = {'method': method_, + 'path': path_} + if method.lower() in ('put', 'post'): + params['body'] = kw + return self.request(**params) + + setattr(self, key, _internal) + + return getattr(self, key) + class HttpClient(Client): def __init__(self, endpoint=None, token=None): diff --git a/keystonelight/service.py b/keystonelight/service.py index a4c46e9e04..536a2a0578 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -1,7 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# this is the web service frontend - import json import logging @@ -15,6 +11,41 @@ from keystonelight import utils from keystonelight import wsgi +HIGH_LEVEL_CALLS = { + 'authenticate': ('POST', '/tokens'), + 'get_tenants': ('GET', '/user/%(user_id)s/tenants'), + 'get_user': ('GET', '/user/%(user_id)s'), + 'get_tenant': ('GET', '/tenant/%(tenant_id)s'), + 'get_tenant_by_name': ('GET', '/tenant_name/%(tenant_name)s'), + 'get_extras': ('GET', '/extras/%(tenant_id)s-%(user_id)s'), + 'get_token': ('GET', '/token/%(token_id)s'), + } + +# NOTE(termie): creates are seperate from updates to allow duplication checks +LOW_LEVEL_CALLS = { + # tokens + 'create_token': ('POST', '/token'), + 'delete_token': ('DELETE', '/token/%(token_id)s'), + # users + 'create_user': ('POST', '/user'), + 'update_user': ('PUT', '/user/%(user_id)s'), + 'delete_user': ('DELETE', '/user/%(user_id)s'), + # tenants + 'create_tenant': ('POST', '/tenant'), + 'update_tenant': ('PUT', '/tenant/%(tenant_id)s'), + 'delete_tenant': ('DELETE', '/tenant/%(tenant_id)s'), + # extras + # NOTE(termie): these separators are probably going to bite us eventually + 'create_extras': ('POST', '/extras'), + 'update_extras': ('PUT', '/extras/%(tenant_id)s-%(user_id)s'), + 'delete_extras': ('DELETE', '/extras/%(tenant_id)s-%(user_id)s'), + } + + +URLMAP = HIGH_LEVEL_CALLS.copy() +URLMAP.update(LOW_LEVEL_CALLS) + + class BaseApplication(wsgi.Application): @webob.dec.wsgify def __call__(self, req): @@ -81,10 +112,11 @@ class IdentityController(BaseApplication): logging.debug('TOKEN: %s', token_ref) return token_ref - def get_tenants(self, context): + def get_tenants(self, context, user_id=None): token_id = context.get('token_id') token_ref = self.token_api.get_token(context, token_id) assert token_ref + assert token_ref['user']['id'] == user_id tenants_ref = [] for tenant_id in token_ref['user']['tenants']: tenants_ref.append(self.identity_api.get_tenant(context, @@ -98,20 +130,31 @@ class Router(wsgi.Router): self.options = options self.identity_controller = IdentityController(options) self.token_controller = TokenController(options) - mapper = routes.Mapper() - mapper.connect('/tokens', - controller=self.identity_controller, - action='authenticate') - mapper.connect('/tokens/{token_id}', - controller=self.token_controller, - action='revoke_token', - conditions=dict(method=['DELETE'])) - mapper.connect("/tenants", - controller=self.identity_controller, - action="get_tenants", - conditions=dict(method=["GET"])) + + mapper = self._build_map(URLMAP) super(Router, self).__init__(mapper) + def _build_map(self, urlmap): + """Build a routes.Mapper based on URLMAP.""" + mapper = routes.Mapper() + for k, v in urlmap.iteritems(): + # NOTE(termie): hack + if 'token' in k: + controller = self.token_controller + else: + controller = self.identity_controller + action = k + method, path = v + path = path.replace('%(', '{').replace(')s', '}') + print path + + mapper.connect(path, + controller=controller, + action=action, + conditions=dict(method=[method])) + + return mapper + def app_factory(global_conf, **local_conf): conf = global_conf.copy() diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 21b2578138..885d43140d 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -51,7 +51,7 @@ class IdentityApi(test.TestCase): post_data = {'user_id': self.user_foo['id'], 'tenant_id': self.tenant_bar['id'], 'password': self.user_foo['password']} - resp = c.post('/tokens', body=post_data) + resp = c.authenticate(**post_data) data = json.loads(resp.body) self.assertEquals(self.user_foo['id'], data['user']['id']) self.assertEquals(self.tenant_bar['id'], data['tenant']['id']) @@ -61,7 +61,7 @@ class IdentityApi(test.TestCase): c = client.TestClient(self.app) post_data = {'user_id': self.user_foo['id'], 'password': self.user_foo['password']} - resp = c.post('/tokens', body=post_data) + resp = c.authenticate(**post_data) data = json.loads(resp.body) self.assertEquals(self.user_foo['id'], data['user']['id']) self.assertEquals(None, data['tenant']) @@ -70,6 +70,6 @@ class IdentityApi(test.TestCase): def test_get_tenants(self): token = self._login() c = client.TestClient(self.app, token['id']) - resp = c.get('/tenants') + resp = c.get_tenants(user_id=self.user_foo['id']) data = json.loads(resp.body) self.assertDictEquals(self.tenant_bar, data[0]) From f2e73bc9b20b26947980067bcf95c9989e37907d Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 09:33:17 -0800 Subject: [PATCH 057/334] re-indent service.py --- keystonelight/service.py | 169 +++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index 536a2a0578..682f5151ed 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -47,116 +47,115 @@ URLMAP.update(LOW_LEVEL_CALLS) class BaseApplication(wsgi.Application): - @webob.dec.wsgify - def __call__(self, req): - arg_dict = req.environ['wsgiorg.routing_args'][1] - action = arg_dict['action'] - del arg_dict['action'] - del arg_dict['controller'] - logging.debug('arg_dict: %s', arg_dict) + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + del arg_dict['controller'] + logging.debug('arg_dict: %s', arg_dict) - context = req.environ.get('openstack.context', {}) - # allow middleware up the stack to override the params - params = {} - if 'openstack.params' in req.environ: - params = req.environ['openstack.params'] - params.update(arg_dict) + context = req.environ.get('openstack.context', {}) + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + params.update(arg_dict) - # TODO(termie): do some basic normalization on methods - method = getattr(self, action) + # TODO(termie): do some basic normalization on methods + method = getattr(self, action) - # NOTE(vish): make sure we have no unicode keys for py2.6. - params = dict([(str(k), v) for (k, v) in params.iteritems()]) - result = method(context, **params) + # NOTE(vish): make sure we have no unicode keys for py2.6. + params = dict([(str(k), v) for (k, v) in params.iteritems()]) + result = method(context, **params) - if result is None or type(result) is str or type(result) is unicode: - return result + if result is None or type(result) is str or type(result) is unicode: + return result - return json.dumps(result) + return json.dumps(result) class TokenController(BaseApplication): - """Validate and pass through calls to TokenManager.""" + """Validate and pass through calls to TokenManager.""" - def __init__(self, options): - self.token_api = token.Manager(options=options) - self.options = options + def __init__(self, options): + self.token_api = token.Manager(options=options) + self.options = options - def validate_token(self, context, token_id): - token_info = self.token_api.validate_token(context, token_id) - if not token_info: - raise webob.exc.HTTPUnauthorized() - return token_info + def validate_token(self, context, token_id): + token_info = self.token_api.validate_token(context, token_id) + if not token_info: + raise webob.exc.HTTPUnauthorized() + return token_info class IdentityController(BaseApplication): - """Validate and pass calls through to IdentityManager. + """Validate and pass calls through to IdentityManager. - IdentityManager will also pretty much just pass calls through to - a specific driver. - """ + IdentityManager will also pretty much just pass calls through to + a specific driver. + """ - def __init__(self, options): - self.identity_api = identity.Manager(options=options) - self.token_api = token.Manager(options=options) - self.options = options + def __init__(self, options): + self.identity_api = identity.Manager(options=options) + self.token_api = token.Manager(options=options) + self.options = options - def authenticate(self, context, **kwargs): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - context, **kwargs) - # TODO(termie): strip password from return values - token_ref = self.token_api.create_token(context, - dict(tenant=tenant_ref, - user=user_ref, - extras=extras_ref)) - logging.debug('TOKEN: %s', token_ref) - return token_ref + def authenticate(self, context, **kwargs): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + context, **kwargs) + # TODO(termie): strip password from return values + token_ref = self.token_api.create_token(context, + dict(tenant=tenant_ref, + user=user_ref, + extras=extras_ref)) + logging.debug('TOKEN: %s', token_ref) + return token_ref - def get_tenants(self, context, user_id=None): - token_id = context.get('token_id') - token_ref = self.token_api.get_token(context, token_id) - assert token_ref - assert token_ref['user']['id'] == user_id - tenants_ref = [] - for tenant_id in token_ref['user']['tenants']: - tenants_ref.append(self.identity_api.get_tenant(context, - tenant_id)) + def get_tenants(self, context, user_id=None): + token_id = context.get('token_id') + token_ref = self.token_api.get_token(context, token_id) + assert token_ref + assert token_ref['user']['id'] == user_id + tenants_ref = [] + for tenant_id in token_ref['user']['tenants']: + tenants_ref.append(self.identity_api.get_tenant(context, + tenant_id)) - return tenants_ref + return tenants_ref class Router(wsgi.Router): - def __init__(self, options): - self.options = options - self.identity_controller = IdentityController(options) - self.token_controller = TokenController(options) + def __init__(self, options): + self.options = options + self.identity_controller = IdentityController(options) + self.token_controller = TokenController(options) - mapper = self._build_map(URLMAP) - super(Router, self).__init__(mapper) + mapper = self._build_map(URLMAP) + super(Router, self).__init__(mapper) - def _build_map(self, urlmap): - """Build a routes.Mapper based on URLMAP.""" - mapper = routes.Mapper() - for k, v in urlmap.iteritems(): - # NOTE(termie): hack - if 'token' in k: - controller = self.token_controller - else: - controller = self.identity_controller - action = k - method, path = v - path = path.replace('%(', '{').replace(')s', '}') - print path + def _build_map(self, urlmap): + """Build a routes.Mapper based on URLMAP.""" + mapper = routes.Mapper() + for k, v in urlmap.iteritems(): + # NOTE(termie): hack + if 'token' in k: + controller = self.token_controller + else: + controller = self.identity_controller + action = k + method, path = v + path = path.replace('%(', '{').replace(')s', '}') - mapper.connect(path, - controller=controller, - action=action, - conditions=dict(method=[method])) + mapper.connect(path, + controller=controller, + action=action, + conditions=dict(method=[method])) - return mapper + return mapper def app_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - return Router(conf) + conf = global_conf.copy() + conf.update(local_conf) + return Router(conf) From 6c84c1bf6a80399e7d1c9695eb01b8ba9fad1fc6 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 09:54:24 -0800 Subject: [PATCH 058/334] reorg --- keystonelight/keystone_compat.py | 106 +++++++++++++++---------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index f751a905ef..c0d5cba82f 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -136,6 +136,59 @@ class KeystoneController(service.BaseApplication): return self._format_authenticate(token_ref, catalog_ref) + #admin-only + def validate_token(self, context, token_id, belongs_to=None): + """Check that a token is valid. + + Optionally, also ensure that it is owned by a specific tenant. + + """ + assert context['is_admin'] + + token_ref = self.token_api.get_token(context=context, + token_id=token_id) + if belongs_to: + assert token_ref['tenant']['id'] == belongs_to + return self._format_token(token_ref) + + def tenants_for_token(self, context): + """Get valid tenants for token based on token used to authenticate. + + Pulls the token from the context, validates it and gets the valid + tenants for the user in the token. + + Doesn't care about token scopedness. + + """ + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + assert token_ref is not None + + user_ref = token_ref['user'] + tenant_refs = [] + for tenant_id in user_ref['tenants']: + tenant_refs.append(self.identity_api.get_tenant( + context=context, + tenant_id=tenant_id)) + return self._format_tenants_for_token(tenant_refs) + + def _format_token(self, token_ref): + user_ref = token_ref['user'] + extras_ref = token_ref['extras'] + o = {'access': {'token': {'id': token_ref['id'], + 'expires': token_ref['expires'] + }, + 'user': {'id': user_ref['id'], + 'name': user_ref['name'], + 'roles': extras_ref['roles'] or [], + 'roles_links': extras_ref['roles_links'] or [] + } + } + } + if 'tenant' in token_ref: + o['access']['token']['tenant'] = token_ref['tenant'] + return o + def _format_authenticate(self, token_ref, catalog_ref): o = self._format_token(token_ref) o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) @@ -185,59 +238,6 @@ class KeystoneController(service.BaseApplication): return services.values() - #admin-only - def validate_token(self, context, token_id, belongs_to=None): - """Check that a token is valid. - - Optionally, also ensure that it is owned by a specific tenant. - - """ - assert context['is_admin'] - - token_ref = self.token_api.get_token(context=context, - token_id=token_id) - if belongs_to: - assert token_ref['tenant']['id'] == belongs_to - return self._format_token(token_ref) - - def _format_token(self, token_ref): - user_ref = token_ref['user'] - extras_ref = token_ref['extras'] - o = {'access': {'token': {'id': token_ref['id'], - 'expires': token_ref['expires'] - }, - 'user': {'id': user_ref['id'], - 'name': user_ref['name'], - 'roles': extras_ref['roles'] or [], - 'roles_links': extras_ref['roles_links'] or [] - } - } - } - if 'tenant' in token_ref: - o['access']['token']['tenant'] = token_ref['tenant'] - return o - - def tenants_for_token(self, context): - """Get valid tenants for token based on token used to authenticate. - - Pulls the token from the context, validates it and gets the valid - tenants for the user in the token. - - Doesn't care about token scopedness. - - """ - token_ref = self.token_api.get_token(context=context, - token_id=context['token_id']) - assert token_ref is not None - - user_ref = token_ref['user'] - tenant_refs = [] - for tenant_id in user_ref['tenants']: - tenant_refs.append(self.identity_api.get_tenant( - context=context, - tenant_id=tenant_id)) - return self._format_tenants_for_token(tenant_refs) - def _format_tenants_for_token(self, tenant_refs): o = {'tenants': tenant_refs, 'tenants_links': []} From adbbe0147e9e726db2dc6f2c2d4e446fa589c5ba Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 09:57:24 -0800 Subject: [PATCH 059/334] use the keystone app in the conf --- bin/keystone | 4 ++-- etc/default.conf | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 etc/default.conf diff --git a/bin/keystone b/bin/keystone index f353e8f9b6..f9db597953 100755 --- a/bin/keystone +++ b/bin/keystone @@ -23,11 +23,11 @@ from keystonelight import wsgi if __name__ == '__main__': default_conf = os.path.join(possible_topdir, 'etc', - 'keystone.conf') + 'default.conf') logging.getLogger().setLevel(logging.DEBUG) conf = len(sys.argv) > 1 and sys.argv[1] or default_conf - app = deploy.loadapp('config:%s' % conf) + app = deploy.loadapp('config:%s' % conf, name='keystone') server = wsgi.Server() server.start(app, int(app.options['public_port'])) server.start(app, int(app.options['admin_port'])) diff --git a/etc/default.conf b/etc/default.conf new file mode 100644 index 0000000000..4622823964 --- /dev/null +++ b/etc/default.conf @@ -0,0 +1,30 @@ +[DEFAULT] +catalog_driver = keystonelight.backends.kvs.KvsCatalog +identity_driver = keystonelight.backends.kvs.KvsIdentity +token_driver = keystonelight.backends.kvs.KvsToken +public_port = 5000 +admin_token = ADMIN + +[filter:debug] +paste.filter_factory = keystonelight.wsgi:Debug.factory + +[filter:token_auth] +paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory + +[filter:admin_token_auth] +paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory + +[filter:json_body] +paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory + +[app:keystone_compat] +paste.app_factory = keystonelight.keystone_compat:app_factory + +[app:keystonelight] +paste.app_factory = keystonelight.service:app_factory + +[pipeline:keystone] +pipeline = token_auth admin_token_auth json_body debug keystone + +[pipeline:main] +pipeline = token_auth admin_token_auth json_body debug keystonelight From e8f72ed91488e1736c65dca467827c4a1adf215b Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:05:50 -0800 Subject: [PATCH 060/334] accept data as kwargs for crud --- keystonelight/identity.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 1b2a2f30ac..c741a0707b 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # these will be the basic data types for tenants and users # backends will make use of them to return something that conforms to their # apis @@ -34,28 +32,28 @@ class Manager(object): return self.driver.get_extras(user_id, tenant_id) # CRUD operations - def create_user(self, context, user_id, data): + def create_user(self, context, user_id, **data): return self.driver.create_user(user_id, data) - def update_user(self, context, user_id, data): + def update_user(self, context, user_id, **data): return self.driver.update_user(user_id, data) def delete_user(self, context, user_id): return self.driver.delete_user(user_id) - def create_tenant(self, context, tenant_id, data): + def create_tenant(self, context, tenant_id, **data): return self.driver.create_tenant(tenant_id, data) - def update_tenant(self, context, tenant_id, data): + def update_tenant(self, context, tenant_id, **data): return self.driver.update_tenant(tenant_id, data) def delete_tenant(self, context, tenant_id): return self.driver.delete_tenant(tenant_id) - def create_extras(self, context, user_id, tenant_id, data): + def create_extras(self, context, user_id, tenant_id, **data): return self.driver.create_extras(user_id, tenant_id, data) - def update_extras(self, context, user_id, tenant_id, data): + def update_extras(self, context, user_id, tenant_id, **data): return self.driver.update_extras(user_id, tenant_id, data) def delete_extras(self, context, user_id, tenant_id): From 91059352640965765afd2e97ac37656e445927cf Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:08:35 -0800 Subject: [PATCH 061/334] don't pep8 swp files --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 2f6fe3bab5..621a6efa50 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -85,7 +85,7 @@ function run_pep8 { ignore_files="*eventlet-patch:*pip-requires" ignore_dirs="*ajaxterm*" GLOBIGNORE="$ignore_scripts:$ignore_files:$ignore_dirs" - srcfiles=`find bin -type f` + srcfiles=`find bin -type f ! -name .*.swp` srcfiles+=" keystonelight" # Just run PEP8 in current environment ${wrapper} pep8 --repeat --show-pep8 --show-source \ From 2d154828e7564f4025436d40cd5183c5f93b4273 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:08:51 -0800 Subject: [PATCH 062/334] re-indent identity.py --- keystonelight/identity.py | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index c741a0707b..3423c683de 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -7,54 +7,54 @@ from keystonelight import utils class Manager(object): - def __init__(self, options): - self.driver = utils.import_object(options['identity_driver'], - options=options) - self.options = options + def __init__(self, options): + self.driver = utils.import_object(options['identity_driver'], + options=options) + self.options = options - def authenticate(self, context, **kwargs): - """Passthru authentication to the identity driver. + def authenticate(self, context, **kwargs): + """Passthru authentication to the identity driver. - This call will basically just result in getting a token. - """ - return self.driver.authenticate(**kwargs) + This call will basically just result in getting a token. + """ + return self.driver.authenticate(**kwargs) - def get_user(self, context, user_id): - return self.driver.get_user(user_id) + def get_user(self, context, user_id): + return self.driver.get_user(user_id) - def get_tenant(self, context, tenant_id): - return self.driver.get_tenant(tenant_id) + def get_tenant(self, context, tenant_id): + return self.driver.get_tenant(tenant_id) - def get_tenant_by_name(self, context, tenant_name): - return self.driver.get_tenant_by_name(tenant_name) + def get_tenant_by_name(self, context, tenant_name): + return self.driver.get_tenant_by_name(tenant_name) - def get_extras(self, context, user_id, tenant_id): - return self.driver.get_extras(user_id, tenant_id) + def get_extras(self, context, user_id, tenant_id): + return self.driver.get_extras(user_id, tenant_id) - # CRUD operations - def create_user(self, context, user_id, **data): - return self.driver.create_user(user_id, data) + # CRUD operations + def create_user(self, context, user_id, **data): + return self.driver.create_user(user_id, data) - def update_user(self, context, user_id, **data): - return self.driver.update_user(user_id, data) + def update_user(self, context, user_id, **data): + return self.driver.update_user(user_id, data) - def delete_user(self, context, user_id): - return self.driver.delete_user(user_id) + def delete_user(self, context, user_id): + return self.driver.delete_user(user_id) - def create_tenant(self, context, tenant_id, **data): - return self.driver.create_tenant(tenant_id, data) + def create_tenant(self, context, tenant_id, **data): + return self.driver.create_tenant(tenant_id, data) - def update_tenant(self, context, tenant_id, **data): - return self.driver.update_tenant(tenant_id, data) + def update_tenant(self, context, tenant_id, **data): + return self.driver.update_tenant(tenant_id, data) - def delete_tenant(self, context, tenant_id): - return self.driver.delete_tenant(tenant_id) + def delete_tenant(self, context, tenant_id): + return self.driver.delete_tenant(tenant_id) - def create_extras(self, context, user_id, tenant_id, **data): - return self.driver.create_extras(user_id, tenant_id, data) + def create_extras(self, context, user_id, tenant_id, **data): + return self.driver.create_extras(user_id, tenant_id, data) - def update_extras(self, context, user_id, tenant_id, **data): - return self.driver.update_extras(user_id, tenant_id, data) + def update_extras(self, context, user_id, tenant_id, **data): + return self.driver.update_extras(user_id, tenant_id, data) - def delete_extras(self, context, user_id, tenant_id): - return self.driver.delete_extras(user_id, tenant_id) + def delete_extras(self, context, user_id, tenant_id): + return self.driver.delete_extras(user_id, tenant_id) From 2c7770fc0d1ef0f55a2e96c8ef6086a2f7d29937 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:23:05 -0800 Subject: [PATCH 063/334] add test for create user and get user --- keystonelight/identity.py | 12 ++++++------ keystonelight/service.py | 9 +++++++++ tests/test_identity_api.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 3423c683de..ac5517b6ac 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -32,28 +32,28 @@ class Manager(object): return self.driver.get_extras(user_id, tenant_id) # CRUD operations - def create_user(self, context, user_id, **data): + def create_user(self, context, user_id, data): return self.driver.create_user(user_id, data) - def update_user(self, context, user_id, **data): + def update_user(self, context, user_id, data): return self.driver.update_user(user_id, data) def delete_user(self, context, user_id): return self.driver.delete_user(user_id) - def create_tenant(self, context, tenant_id, **data): + def create_tenant(self, context, tenant_id, data): return self.driver.create_tenant(tenant_id, data) - def update_tenant(self, context, tenant_id, **data): + def update_tenant(self, context, tenant_id, data): return self.driver.update_tenant(tenant_id, data) def delete_tenant(self, context, tenant_id): return self.driver.delete_tenant(tenant_id) - def create_extras(self, context, user_id, tenant_id, **data): + def create_extras(self, context, user_id, tenant_id, data): return self.driver.create_extras(user_id, tenant_id, data) - def update_extras(self, context, user_id, tenant_id, **data): + def update_extras(self, context, user_id, tenant_id, data): return self.driver.update_extras(user_id, tenant_id, data) def delete_extras(self, context, user_id, tenant_id): diff --git a/keystonelight/service.py b/keystonelight/service.py index 682f5151ed..272e08227c 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -1,5 +1,6 @@ import json import logging +import uuid import routes import webob.dec @@ -124,6 +125,14 @@ class IdentityController(BaseApplication): return tenants_ref + def create_user(self, context, **kw): + user_id = uuid.uuid4().hex + kw['id'] = user_id + return self.identity_api.create_user(context, user_id=user_id, data=kw) + + def get_user(self, context, user_id): + return self.identity_api.get_user(context, user_id=user_id) + class Router(wsgi.Router): def __init__(self, options): diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 885d43140d..8799ff361c 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -73,3 +73,16 @@ class IdentityApi(test.TestCase): resp = c.get_tenants(user_id=self.user_foo['id']) data = json.loads(resp.body) self.assertDictEquals(self.tenant_bar, data[0]) + + def test_create_user(self): + token_id = self.options['admin_token'] + c = client.TestClient(self.app, token=token_id) + user_ref = models.User() + resp = c.create_user(**user_ref) + data = json.loads(resp.body) + self.assert_(data['id']) + + new_resp = c.get_user(user_id=data['id']) + new_data = json.loads(new_resp.body) + + self.assertDictEquals(data, new_data) From 8ff5606b3d63a6ab31e69f5605b1fbe4ce59dda4 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:24:45 -0800 Subject: [PATCH 064/334] add test for create user and get user --- keystonelight/service.py | 8 ++++++-- tests/test_identity_api.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index 272e08227c..758d87ba26 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -125,13 +125,17 @@ class IdentityController(BaseApplication): return tenants_ref + def get_user(self, context, user_id): + return self.identity_api.get_user(context, user_id=user_id) + def create_user(self, context, **kw): user_id = uuid.uuid4().hex kw['id'] = user_id return self.identity_api.create_user(context, user_id=user_id, data=kw) - def get_user(self, context, user_id): - return self.identity_api.get_user(context, user_id=user_id) + def delete_user(self, context, user_id): + return self.identity_api.delete_user(context, user_id=user_id) + class Router(wsgi.Router): diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 8799ff361c..8700b7bce3 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -82,7 +82,12 @@ class IdentityApi(test.TestCase): data = json.loads(resp.body) self.assert_(data['id']) - new_resp = c.get_user(user_id=data['id']) - new_data = json.loads(new_resp.body) + get_resp = c.get_user(user_id=data['id']) + get_data = json.loads(get_resp.body) - self.assertDictEquals(data, new_data) + self.assertDictEquals(data, get_data) + + del_resp = c.delete_user(user_id=data['id']) + self.assertEquals(del_resp.body, '') + + delget_resp = c.get_user(user_id=data['id']) From d0009db73564aee8eb8cd34df21457e7d3d8c851 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:29:56 -0800 Subject: [PATCH 065/334] add crud tests --- keystonelight/backends/kvs.py | 4 ++++ keystonelight/service.py | 1 - tests/test_identity_api.py | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index e7fa369fbb..f71f7bbd1b 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -56,6 +56,10 @@ class KvsIdentity(object): self.db.set('user-%s' % id, user) return user + def delete_user(self, id): + self.db.delete('user-%s' % id) + return None + def create_tenant(self, id, tenant): self.db.set('tenant-%s' % id, tenant) self.db.set('tenant_name-%s' % tenant['name'], tenant) diff --git a/keystonelight/service.py b/keystonelight/service.py index 758d87ba26..a089e94fed 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -137,7 +137,6 @@ class IdentityController(BaseApplication): return self.identity_api.delete_user(context, user_id=user_id) - class Router(wsgi.Router): def __init__(self, options): self.options = options diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 8700b7bce3..ff7291d83c 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -74,7 +74,7 @@ class IdentityApi(test.TestCase): data = json.loads(resp.body) self.assertDictEquals(self.tenant_bar, data[0]) - def test_create_user(self): + def test_crud_user(self): token_id = self.options['admin_token'] c = client.TestClient(self.app, token=token_id) user_ref = models.User() @@ -91,3 +91,6 @@ class IdentityApi(test.TestCase): self.assertEquals(del_resp.body, '') delget_resp = c.get_user(user_id=data['id']) + self.assertEquals(delget_resp.body, '') + # TODO(termie): we should probably return not founds instead of None + #self.assertEquals(delget_resp.status, '404 Not Found') From 54f32f96fe43302c9e4129b6e1ca1954882f7555 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:32:40 -0800 Subject: [PATCH 066/334] add crud tests --- keystonelight/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index a089e94fed..586853dcc7 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -166,7 +166,6 @@ class Router(wsgi.Router): return mapper - def app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) From 7035e4aa783e6423f1ba691c19f596386c7e12e4 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:33:13 -0800 Subject: [PATCH 067/334] add crud tests --- keystonelight/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keystonelight/service.py b/keystonelight/service.py index 586853dcc7..a089e94fed 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -166,6 +166,7 @@ class Router(wsgi.Router): return mapper + def app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) From f8e6fae92f9defb23ee80abea6126d6753574393 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:37:24 -0800 Subject: [PATCH 068/334] oops, forgot update in crud --- keystonelight/backends/kvs.py | 4 ++++ keystonelight/service.py | 5 +++++ tests/test_identity_api.py | 8 ++++++++ 3 files changed, 17 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index f71f7bbd1b..7892e3cd05 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -56,6 +56,10 @@ class KvsIdentity(object): self.db.set('user-%s' % id, user) return user + def update_user(self, id, user): + self.db.set('user-%s' % id, user) + return user + def delete_user(self, id): self.db.delete('user-%s' % id) return None diff --git a/keystonelight/service.py b/keystonelight/service.py index a089e94fed..c65102a327 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -133,6 +133,11 @@ class IdentityController(BaseApplication): kw['id'] = user_id return self.identity_api.create_user(context, user_id=user_id, data=kw) + def update_user(self, context, user_id, **kw): + kw['id'] = user_id + kw.pop('user_id', None) + return self.identity_api.update_user(context, user_id=user_id, data=kw) + def delete_user(self, context, user_id): return self.identity_api.delete_user(context, user_id=user_id) diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index ff7291d83c..8d3e84c67b 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -87,6 +87,14 @@ class IdentityApi(test.TestCase): self.assertDictEquals(data, get_data) + update_resp = c.update_user(user_id=data['id'], + id=data['id'], + password='foo') + update_data = json.loads(update_resp.body) + + self.assertEquals(data['id'], update_data['id']) + self.assertEquals('foo', update_data['password']) + del_resp = c.delete_user(user_id=data['id']) self.assertEquals(del_resp.body, '') From 3ab9d87465d7aa96c7b3da4d6425540bc67c1a5b Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 10:51:17 -0800 Subject: [PATCH 069/334] add tenant crud --- keystonelight/backends/kvs.py | 14 +++++++++++ keystonelight/service.py | 22 ++++++++++++++++++ tests/test_identity_api.py | 44 +++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 7892e3cd05..d55541396e 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -69,6 +69,20 @@ class KvsIdentity(object): self.db.set('tenant_name-%s' % tenant['name'], tenant) return tenant + def update_tenant(self, id, tenant): + # get the old name and delete it too + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def delete_tenant(self, id): + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.delete('tenant-%s' % id) + return None + def create_extras(self, user_id, tenant_id, extras): self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) return extras diff --git a/keystonelight/service.py b/keystonelight/service.py index c65102a327..2d5556ef9f 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -141,6 +141,28 @@ class IdentityController(BaseApplication): def delete_user(self, context, user_id): return self.identity_api.delete_user(context, user_id=user_id) + def get_tenant(self, context, tenant_id): + return self.identity_api.get_tenant(context, tenant_id=tenant_id) + + def get_tenant_by_name(self, context, tenant_name): + return self.identity_api.get_tenant_by_name( + context, tenant_name=tenant_name) + + def create_tenant(self, context, **kw): + tenant_id = uuid.uuid4().hex + kw['id'] = tenant_id + return self.identity_api.create_tenant( + context, tenant_id=tenant_id, data=kw) + + def update_tenant(self, context, tenant_id, **kw): + kw['id'] = tenant_id + kw.pop('tenant_id', None) + return self.identity_api.update_tenant( + context, tenant_id=tenant_id, data=kw) + + def delete_tenant(self, context, tenant_id): + return self.identity_api.delete_tenant(context, tenant_id=tenant_id) + class Router(wsgi.Router): def __init__(self, options): diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 8d3e84c67b..29b6da3b4d 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -102,3 +102,47 @@ class IdentityApi(test.TestCase): self.assertEquals(delget_resp.body, '') # TODO(termie): we should probably return not founds instead of None #self.assertEquals(delget_resp.status, '404 Not Found') + + def test_crud_tenant(self): + token_id = self.options['admin_token'] + c = client.TestClient(self.app, token=token_id) + tenant_ref = models.Tenant(name='BAZ') + resp = c.create_tenant(**tenant_ref) + data = json.loads(resp.body) + self.assert_(data['id']) + + get_resp = c.get_tenant(tenant_id=data['id']) + get_data = json.loads(get_resp.body) + self.assertDictEquals(data, get_data) + + getname_resp = c.get_tenant_by_name(tenant_name=data['name']) + getname_data = json.loads(getname_resp.body) + self.assertDictEquals(data, getname_data) + + update_resp = c.update_tenant(tenant_id=data['id'], + id=data['id'], + name='NEWBAZ') + update_data = json.loads(update_resp.body) + + self.assertEquals(data['id'], update_data['id']) + self.assertEquals('NEWBAZ', update_data['name']) + + # make sure we can't get the old name + getname_resp = c.get_tenant_by_name(tenant_name=data['name']) + self.assertEquals(getname_resp.body, '') + + # but can get the new name + getname_resp = c.get_tenant_by_name(tenant_name=update_data['name']) + getname_data = json.loads(getname_resp.body) + self.assertDictEquals(update_data, getname_data) + + del_resp = c.delete_tenant(tenant_id=data['id']) + self.assertEquals(del_resp.body, '') + + delget_resp = c.get_tenant(tenant_id=data['id']) + self.assertEquals(delget_resp.body, '') + + delgetname_resp = c.get_tenant_by_name(tenant_name=update_data['name']) + self.assertEquals(delgetname_resp.body, '') + # TODO(termie): we should probably return not founds instead of None + #self.assertEquals(delget_resp.status, '404 Not Found') From 2545907561e3a0c943c204b0eec491555dd5c537 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 11:12:26 -0800 Subject: [PATCH 070/334] add tests for extras --- keystonelight/backends/kvs.py | 8 ++++++++ keystonelight/service.py | 21 +++++++++++++++++++++ tests/test_identity_api.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index d55541396e..982be99f6e 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -87,6 +87,14 @@ class KvsIdentity(object): self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) return extras + def update_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + + def delete_extras(self, user_id, tenant_id): + self.db.delete('extras-%s-%s' % (tenant_id, user_id)) + return None + # Private CRUD for testing def _create_user(self, id, user): self.db.set('user-%s' % id, user) diff --git a/keystonelight/service.py b/keystonelight/service.py index 2d5556ef9f..659ad8b633 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -125,6 +125,7 @@ class IdentityController(BaseApplication): return tenants_ref + # crud api def get_user(self, context, user_id): return self.identity_api.get_user(context, user_id=user_id) @@ -163,6 +164,26 @@ class IdentityController(BaseApplication): def delete_tenant(self, context, tenant_id): return self.identity_api.delete_tenant(context, tenant_id=tenant_id) + def get_extras(self, context, user_id, tenant_id): + return self.identity_api.get_extras( + context, user_id=user_id, tenant_id=tenant_id) + + def create_extras(self, context, **kw): + user_id = kw.pop('user_id') + tenant_id = kw.pop('tenant_id') + return self.identity_api.create_extras( + context, user_id=user_id, tenant_id=tenant_id, data=kw) + + def update_extras(self, context, user_id, tenant_id, **kw): + kw.pop('user_id', None) + kw.pop('tenant_id', None) + return self.identity_api.update_extras( + context, user_id=user_id, tenant_id=tenant_id, data=kw) + + def delete_extras(self, context, user_id, tenant_id): + return self.identity_api.delete_extras( + context, user_id=user_id, tenant_id=tenant_id) + class Router(wsgi.Router): def __init__(self, options): diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 29b6da3b4d..22668bac46 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -146,3 +146,33 @@ class IdentityApi(test.TestCase): self.assertEquals(delgetname_resp.body, '') # TODO(termie): we should probably return not founds instead of None #self.assertEquals(delget_resp.status, '404 Not Found') + + def test_crud_extras(self): + token_id = self.options['admin_token'] + user_id = 'foo' + tenant_id = 'bar' + c = client.TestClient(self.app, token=token_id) + extras_ref = dict(baz='qaz') + resp = c.create_extras(user_id=user_id, tenant_id=tenant_id, **extras_ref) + data = json.loads(resp.body) + self.assertEquals(data['baz'], 'qaz') + + get_resp = c.get_extras(user_id=user_id, tenant_id=tenant_id) + get_data = json.loads(get_resp.body) + + self.assertDictEquals(data, get_data) + + update_resp = c.update_extras(user_id=user_id, + tenant_id=tenant_id, + baz='WAZ') + update_data = json.loads(update_resp.body) + + self.assertEquals('WAZ', update_data['baz']) + + del_resp = c.delete_extras(user_id=user_id, tenant_id=tenant_id) + self.assertEquals(del_resp.body, '') + + delget_resp = c.get_extras(user_id=user_id, tenant_id=tenant_id) + self.assertEquals(delget_resp.body, '') + # TODO(termie): we should probably return not founds instead of None + #self.assertEquals(delget_resp.status, '404 Not Found') From 9d998211529fb6de24d7a62718ad2a850f19557e Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 11:46:32 -0800 Subject: [PATCH 071/334] adjust paths and use composite apps --- bin/keystone | 2 +- etc/default.conf | 15 ++++++++----- keystonelight/backends/templated.py | 1 - keystonelight/keystone_compat.py | 6 +++--- keystonelight/service.py | 4 ++++ keystonelight/test.py | 28 ++++++++++++++++++++----- keystonelight/wsgi.py | 1 + tests/keystone_compat_diablo.conf | 13 +++++++++++- tests/keystoneclient_compat_master.conf | 13 +++++++++++- tests/test_keystoneclient_compat.py | 2 +- 10 files changed, 67 insertions(+), 18 deletions(-) diff --git a/bin/keystone b/bin/keystone index f9db597953..8347fe227c 100755 --- a/bin/keystone +++ b/bin/keystone @@ -27,7 +27,7 @@ if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) conf = len(sys.argv) > 1 and sys.argv[1] or default_conf - app = deploy.loadapp('config:%s' % conf, name='keystone') + app = deploy.loadapp('config:%s' % conf) server = wsgi.Server() server.start(app, int(app.options['public_port'])) server.start(app, int(app.options['admin_port'])) diff --git a/etc/default.conf b/etc/default.conf index 4622823964..dc5575d90a 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -17,14 +17,19 @@ paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory -[app:keystone_compat] -paste.app_factory = keystonelight.keystone_compat:app_factory - [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[pipeline:keystone] +[app:keystone] +paste.app_factory = keystonelight.keystone_compat:app_factory + +[pipeline:keystone_api] pipeline = token_auth admin_token_auth json_body debug keystone -[pipeline:main] +[pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight + +[composite:main] +use = egg:Paste#urlmap +/ = keystonelight_api +/v2.0 = keystone_api diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index e3e9b65895..274ba39abc 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -32,7 +32,6 @@ class TemplatedCatalog(object): def __init__(self, options, templates=None): self.options = options - logging.debug('CATALOG PUBLIC PORT BEFORE: %s', options['public_port']) if templates: self.templates = templates diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index c0d5cba82f..63603dbd84 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -21,15 +21,15 @@ class KeystoneRouter(wsgi.Router): mapper.connect('/', controller=self.keystone_controller, action='noop') - mapper.connect('/v2.0/tokens', + mapper.connect('/tokens', controller=self.keystone_controller, action='authenticate', conditions=dict(method=['POST'])) - mapper.connect('/v2.0/tokens/{token_id}', + mapper.connect('/tokens/{token_id}', controller=self.keystone_controller, action='validate_token', conditions=dict(method=['GET'])) - mapper.connect('/v2.0/tenants', + mapper.connect('/tenants', controller=self.keystone_controller, action='tenants_for_token', conditions=dict(method=['GET'])) diff --git a/keystonelight/service.py b/keystonelight/service.py index 659ad8b633..19f85e20cb 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -192,8 +192,12 @@ class Router(wsgi.Router): self.token_controller = TokenController(options) mapper = self._build_map(URLMAP) + mapper.connect('/', controller=self, action='noop') super(Router, self).__init__(mapper) + def noop(self, context, *args, **kw): + return '' + def _build_map(self, urlmap): """Build a routes.Mapper based on URLMAP.""" mapper = routes.Mapper() diff --git a/keystonelight/test.py b/keystonelight/test.py index e32a78ec3a..72cc63ca55 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -102,7 +102,6 @@ class TestCase(unittest.TestCase): # Service catalog tests need to know the port we ran on. port = server.socket_info['socket'][1] self._update_server_options(server, 'public_port', port) - logging.debug('PUBLIC PORT: %s', app.options['public_port']) return server def _update_server_options(self, server, key, value): @@ -112,13 +111,32 @@ class TestCase(unittest.TestCase): """ last = server - while hasattr(last, 'application') or hasattr(last, 'options'): - logging.debug('UPDATE %s', last.__class__) + + applications = [] + + while (hasattr(last, 'applications') + or hasattr(last, 'application') + or hasattr(last, 'options')): + + logging.debug('UPDATE %s: O %s A %s AS %s', + last.__class__, + getattr(last, 'options', None), + getattr(last, 'application', None), + getattr(last, 'applications', None)) if hasattr(last, 'options'): last.options[key] = value - if not hasattr(last, 'application'): + + # NOTE(termie): paste.urlmap.URLMap stores applications in this format + if hasattr(last, 'applications'): + for app in last.applications: + applications.append(app[1]) + + if hasattr(last, 'application'): + last = last.application + elif len(applications): + last = applications.pop() + else: break - last = last.application def client(self, app, *args, **kw): return TestClient(app, *args, **kw) diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index c31052c6f4..a3b4ccaa19 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -31,6 +31,7 @@ import routes.middleware import webob import webob.dec import webob.exc + from paste import deploy diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index 94024df33f..505c980497 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -17,8 +17,19 @@ paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +[app:keystonelight] +paste.app_factory = keystonelight.service:app_factory + [app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory -[pipeline:main] +[pipeline:keystone_api] pipeline = token_auth admin_token_auth json_body debug keystone + +[pipeline:keystonelight_api] +pipeline = token_auth admin_token_auth json_body debug keystonelight + +[composite:main] +use = egg:Paste#urlmap +/ = keystonelight_api +/v2.0 = keystone_api diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index e9861b6b98..545ff26602 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -32,8 +32,19 @@ paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +[app:keystonelight] +paste.app_factory = keystonelight.service:app_factory + [app:keystone] paste.app_factory = keystonelight.keystone_compat:app_factory -[pipeline:main] +[pipeline:keystone_api] pipeline = token_auth admin_token_auth json_body debug keystone + +[pipeline:keystonelight_api] +pipeline = token_auth admin_token_auth json_body debug keystonelight + +[composite:main] +use = egg:Paste#urlmap +/ = keystonelight_api +/v2.0 = keystone_api diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 30854fbb18..7bc8a1ee22 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -71,7 +71,7 @@ class MasterCompatTestCase(CompatTestCase): port = self.server.socket_info['socket'][1] self.options['public_port'] = port # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, + client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, username='foo', password='foo', project_id='bar') From 1335e4c2ea5a9edac23d00a235016d88887fb701 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 15:50:46 -0800 Subject: [PATCH 072/334] cli for adding users, tenants, extras --- bin/keystone | 5 +-- bin/ksl | 75 ++++++++++++++++++++++++++++++++++++++++- etc/default.conf | 7 ++-- keystonelight/client.py | 7 ++-- 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/bin/keystone b/bin/keystone index 8347fe227c..f902c31b1d 100755 --- a/bin/keystone +++ b/bin/keystone @@ -28,7 +28,8 @@ if __name__ == '__main__': conf = len(sys.argv) > 1 and sys.argv[1] or default_conf app = deploy.loadapp('config:%s' % conf) + options = deploy.appconfig('config:%s' % conf) server = wsgi.Server() - server.start(app, int(app.options['public_port'])) - server.start(app, int(app.options['admin_port'])) + server.start(app, int(options['public_port'])) + server.start(app, int(options['admin_port'])) server.wait() diff --git a/bin/ksl b/bin/ksl index e8f2587a4a..46b8913690 100755 --- a/bin/ksl +++ b/bin/ksl @@ -5,9 +5,16 @@ import sys import cli.app import cli.log +from keystonelight import client + DEFAULT_PARAMS = ( - (('--url',), {'dest': 'url', 'action': 'store'}), + (('--config',), {'dest': 'configfile', + 'action': 'store', + 'default': './etc/default.conf'}), + (('--url',), {'dest': 'url', + 'action': 'store', + 'default': 'http://localhost:5000'}), (('--token',), {'dest': 'token', 'action': 'store'}), ) @@ -21,6 +28,20 @@ class BaseApp(cli.log.LoggingApp): for args, kw in DEFAULT_PARAMS: self.add_param(*args, **kw) + def _parse_keyvalues(self, args): + kv = {} + for x in args: + key, value = x.split('=', 1) + # make lists if there are multiple values + if key in kv: + if type(kv) is type(tuple()): + kv[key] = kv[key] + (value,) + else: + kv[key] = (kv[key], value) + else: + kv[key] = value + return kv + class LoadData(BaseApp): def __init__(self, *args, **kw): @@ -33,7 +54,59 @@ class LoadData(BaseApp): pass +class CrudCommands(BaseApp): + ACTION_MAP = {} + + def __init__(self, *args, **kw): + super(CrudCommands, self).__init__(*args, **kw) + self.add_default_params() + self.add_param('action') + self.add_param('keyvalues', nargs='+') + + def main(self): + """Given some keyvalues create the appropriate data in Keystone Light.""" + c = client.HttpClient(self.params.url, token=self.params.token) + action_name = self.ACTION_MAP[self.params.action] + kv = self._parse_keyvalues(self.params.keyvalues) + resp = getattr(c, action_name)(**kv) + print resp + + +class UserCommands(CrudCommands): + ACTION_MAP = {'add': 'create_user', + 'create': 'create_user',} + + +class TenantCommands(CrudCommands): + ACTION_MAP = {'add': 'create_tenant', + 'create': 'create_tenant',} + + +class ExtrasCommands(CrudCommands): + ACTION_MAP = {'add': 'create_extras', + 'create': 'create_extras',} + + +class Auth(BaseApp): + def __init__(self, *args, **kw): + super(Auth, self).__init__(*args, **kw) + self.add_default_params() + self.add_param('keyvalues', nargs='+') + + def main(self): + """Attempt to authenticate against the Keystone Light API.""" + c = client.HttpClient(self.params.url, token=self.params.token) + kv = self._parse_keyvalues(self.params.keyvalues) + resp = c.authenticate(**kv) + print resp + + + CMDS = {'loaddata': LoadData, + 'user': UserCommands, + 'tenant': TenantCommands, + 'extras': ExtrasCommands, + 'auth': Auth, } diff --git a/etc/default.conf b/etc/default.conf index dc5575d90a..176836dd1e 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -3,19 +3,20 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken public_port = 5000 +admin_port = 5001 admin_token = ADMIN [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory +paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory diff --git a/keystonelight/client.py b/keystonelight/client.py index 25128ba33f..f5a78c5bbd 100644 --- a/keystonelight/client.py +++ b/keystonelight/client.py @@ -77,9 +77,10 @@ class HttpClient(Client): if type(body) is type({}): body = json.dumps(body) headers = self._build_headers(headers) - h = httplib.Http() - resp, content = h.request(path, method=method, headers=headers, body=body) - return webob.Response(content, status=resp.status, headerlist=resp.headers) + h = httplib2.Http() + url = '%s%s' % (self.endpoint, path) + resp, content = h.request(url, method=method, headers=headers, body=body) + return webob.Response(content, status=resp.status, headerlist=resp.items()) class TestClient(Client): From 8eea6b390b51f40130a96dd745cc907483a882a4 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 15:51:29 -0800 Subject: [PATCH 073/334] update test conf too --- tests/default.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/default.conf b/tests/default.conf index 220a350f9e..6d420e45e6 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -3,13 +3,14 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken public_port = 5000 +admin_port = 5001 admin_token = ADMIN # config for TemplatedCatalog, using camelCase because I don't want to do # translations for keystone compat catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 +catalog.RegionOne.identity.internalURL = http://localhost:$(admin_port)s/v2.0 catalog.RegionOne.identity.name = 'Identity Service' # fake compute service for now to help novaclient tests work From c8b28b59eda06a95aba3827090c283085358ea1b Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 15:52:25 -0800 Subject: [PATCH 074/334] pep8 --- bin/ksl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/ksl b/bin/ksl index 46b8913690..3354288a68 100755 --- a/bin/ksl +++ b/bin/ksl @@ -74,17 +74,20 @@ class CrudCommands(BaseApp): class UserCommands(CrudCommands): ACTION_MAP = {'add': 'create_user', - 'create': 'create_user',} + 'create': 'create_user', + } class TenantCommands(CrudCommands): ACTION_MAP = {'add': 'create_tenant', - 'create': 'create_tenant',} + 'create': 'create_tenant', + } class ExtrasCommands(CrudCommands): ACTION_MAP = {'add': 'create_extras', - 'create': 'create_extras',} + 'create': 'create_extras', + } class Auth(BaseApp): @@ -101,7 +104,6 @@ class Auth(BaseApp): print resp - CMDS = {'loaddata': LoadData, 'user': UserCommands, 'tenant': TenantCommands, From 776a15920e171c73c5bfc3ec75aa826e7fff6d43 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 16:24:59 -0800 Subject: [PATCH 075/334] users require a name --- keystonelight/backends/kvs.py | 25 +++++++++++-------------- keystonelight/identity.py | 3 +++ keystonelight/keystone_compat.py | 9 ++++++++- tests/test_backend_kvs.py | 6 +++--- tests/test_identity_api.py | 9 +++++---- tests/test_keystone_compat.py | 16 ++++++++-------- tests/test_keystoneclient_compat.py | 8 ++++---- tests/test_novaclient_compat.py | 6 +++--- 8 files changed, 45 insertions(+), 37 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 982be99f6e..763d65fe34 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -49,18 +49,29 @@ class KvsIdentity(object): user_ref = self.db.get('user-%s' % user_id) return user_ref + def get_user_by_name(self, user_name): + user_ref = self.db.get('user_name-%s' % user_name) + return user_ref + def get_extras(self, user_id, tenant_id): return self.db.get('extras-%s-%s' % (tenant_id, user_id)) def create_user(self, id, user): self.db.set('user-%s' % id, user) + self.db.set('user_name-%s' % user['name'], user) return user def update_user(self, id, user): + # get the old name and delete it too + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) self.db.set('user-%s' % id, user) + self.db.set('user_name-%s' % user['name'], user) return user def delete_user(self, id): + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) self.db.delete('user-%s' % id) return None @@ -95,20 +106,6 @@ class KvsIdentity(object): self.db.delete('extras-%s-%s' % (tenant_id, user_id)) return None - # Private CRUD for testing - def _create_user(self, id, user): - self.db.set('user-%s' % id, user) - return user - - def _create_tenant(self, id, tenant): - self.db.set('tenant-%s' % id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant - - def _create_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras - class KvsToken(object): def __init__(self, options, db=None): diff --git a/keystonelight/identity.py b/keystonelight/identity.py index ac5517b6ac..42c73e0543 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -22,6 +22,9 @@ class Manager(object): def get_user(self, context, user_id): return self.driver.get_user(user_id) + def get_user_by_name(self, context, user_name): + return self.driver.get_user_by_name(user_name) + def get_tenant(self, context, tenant_id): return self.driver.get_tenant(tenant_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 63603dbd84..435f26efd9 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -74,6 +74,13 @@ class KeystoneController(service.BaseApplication): password = auth['passwordCredentials'].get('password', '') tenant_name = auth.get('tenantName', None) + if username: + user_ref = self.identity_api.get_user_by_name( + context=context, user_name=username) + user_id = user_ref['id'] + else: + user_id = auth['passwordCredentials'].get('userId', None) + # more compat if tenant_name: tenant_ref = self.identity_api.get_tenant_by_name( @@ -84,7 +91,7 @@ class KeystoneController(service.BaseApplication): (user_ref, tenant_ref, extras_ref) = \ self.identity_api.authenticate(context=context, - user_id=username, + user_id=user_id, password=password, tenant_id=tenant_id) token_ref = self.token_api.create_token(context, diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 3561a13c5b..676b340d2b 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -13,16 +13,16 @@ class KvsIdentity(test.TestCase): self._load_fixtures() def _load_fixtures(self): - self.tenant_bar = self.identity_api._create_tenant( + self.tenant_bar = self.identity_api.create_tenant( 'bar', models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_api._create_user( + self.user_foo = self.identity_api.create_user( 'foo', models.User(id='foo', name='FOO', password='foo2', tenants=[self.tenant_bar['id']])) - self.extras_foobar = self.identity_api._create_extras( + self.extras_foobar = self.identity_api.create_extras( 'foo', 'bar', {'extra': 'extra'}) diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index 22668bac46..a363574491 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -24,16 +24,16 @@ class IdentityApi(test.TestCase): self._load_fixtures() def _load_fixtures(self): - self.tenant_bar = self.identity_backend._create_tenant( + self.tenant_bar = self.identity_backend.create_tenant( 'bar', models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_backend._create_user( + self.user_foo = self.identity_backend.create_user( 'foo', models.User(id='foo', name='FOO', password='foo2', tenants=[self.tenant_bar['id']])) - self.extras_foobar = self.identity_backend._create_extras( + self.extras_foobar = self.identity_backend.create_extras( 'foo', 'bar', {'extra': 'extra'}) @@ -77,7 +77,7 @@ class IdentityApi(test.TestCase): def test_crud_user(self): token_id = self.options['admin_token'] c = client.TestClient(self.app, token=token_id) - user_ref = models.User() + user_ref = models.User(name='FOO') resp = c.create_user(**user_ref) data = json.loads(resp.body) self.assert_(data['id']) @@ -88,6 +88,7 @@ class IdentityApi(test.TestCase): self.assertDictEquals(data, get_data) update_resp = c.update_user(user_id=data['id'], + name='FOO', id=data['id'], password='foo') update_data = json.loads(update_resp.body) diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index a32477ca26..c8768c02c7 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -46,16 +46,16 @@ class CompatTestCase(test.TestCase): os.path.join(self.sampledir, 'auth.json'))) # validate_token call - self.tenant_345 = self.identity_backend._create_tenant( + self.tenant_345 = self.identity_backend.create_tenant( '345', models.Tenant(id='345', name='My Project')) - self.user_123 = self.identity_backend._create_user( + self.user_123 = self.identity_backend.create_user( '123', models.User(id='123', name='jqsmith', tenants=[self.tenant_345['id']], password='password')) - self.extras_123 = self.identity_backend._create_extras( + self.extras_123 = self.identity_backend.create_extras( self.user_123['id'], self.tenant_345['id'], dict(roles=[{'id': '234', 'name': 'compute:admin'}, @@ -79,21 +79,21 @@ class CompatTestCase(test.TestCase): #catalog = json.load(open( # os.path.join(os.path.dirname(__file__), # 'keystone_compat_diablo_sample_catalog.json'))) - #self.catalog_backend._create_catalog(self.user_123['id'], + #self.catalog_backend.create_catalog(self.user_123['id'], # self.tenant_345['id'], # catalog) # tenants_for_token call - self.user_foo = self.identity_backend._create_user( + self.user_foo = self.identity_backend.create_user( 'foo', - models.User(id='foo', tenants=['1234', '3456'])) - self.tenant_1234 = self.identity_backend._create_tenant( + models.User(id='foo', name='FOO', tenants=['1234', '3456'])) + self.tenant_1234 = self.identity_backend.create_tenant( '1234', models.Tenant(id='1234', name='ACME Corp', description='A description ...', enabled=True)) - self.tenant_3456 = self.identity_backend._create_tenant( + self.tenant_3456 = self.identity_backend.create_tenant( '3456', models.Tenant(id='3456', name='Iron Works', diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 7bc8a1ee22..9470ae33db 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -39,18 +39,18 @@ class MasterCompatTestCase(CompatTestCase): self.server = self.serveapp('keystoneclient_compat_master') - self.tenant_bar = self.identity_backend._create_tenant( + self.tenant_bar = self.identity_backend.create_tenant( 'bar', models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_backend._create_user( + self.user_foo = self.identity_backend.create_user( 'foo', models.User(id='foo', name='FOO', tenants=[self.tenant_bar['id']], password='foo')) - self.extras_bar_foo = self.identity_backend._create_extras( + self.extras_bar_foo = self.identity_backend.create_extras( self.user_foo['id'], self.tenant_bar['id'], dict(roles=[], roles_links=[])) @@ -72,7 +72,7 @@ class MasterCompatTestCase(CompatTestCase): self.options['public_port'] = port # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='foo', + username='FOO', password='foo', project_id='bar') client.authenticate() diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 73a9f533ad..bacd875abf 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -40,18 +40,18 @@ class NovaClientCompatMasterTestCase(CompatTestCase): self.server = self.serveapp('keystoneclient_compat_master') - self.tenant_bar = self.identity_backend._create_tenant( + self.tenant_bar = self.identity_backend.create_tenant( 'bar', models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_backend._create_user( + self.user_foo = self.identity_backend.create_user( 'foo', models.User(id='foo', name='FOO', tenants=[self.tenant_bar['id']], password='foo')) - self.extras_bar_foo = self.identity_backend._create_extras( + self.extras_bar_foo = self.identity_backend.create_extras( self.user_foo['id'], self.tenant_bar['id'], dict(roles=[], roles_links=[])) From 90243515b1216a5d5bc36c5fbe5b3645bb7afb76 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 14 Nov 2011 16:37:34 -0800 Subject: [PATCH 076/334] allow setting user_id on create --- keystonelight/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index 19f85e20cb..c8a236f6db 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -130,7 +130,7 @@ class IdentityController(BaseApplication): return self.identity_api.get_user(context, user_id=user_id) def create_user(self, context, **kw): - user_id = uuid.uuid4().hex + user_id = kw.get('id', uuid.uuid4().hex) kw['id'] = user_id return self.identity_api.create_user(context, user_id=user_id, data=kw) @@ -150,7 +150,7 @@ class IdentityController(BaseApplication): context, tenant_name=tenant_name) def create_tenant(self, context, **kw): - tenant_id = uuid.uuid4().hex + tenant_id = kw.get('id', uuid.uuid4().hex) kw['id'] = tenant_id return self.identity_api.create_tenant( context, tenant_id=tenant_id, data=kw) From 17e03b82cb09d97545bd5a45d846d3d98688408d Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 09:43:42 -0800 Subject: [PATCH 077/334] move noop to identity controller --- keystonelight/service.py | 8 ++++---- tools/pip-requires | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index c8a236f6db..0294ba292c 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -102,6 +102,9 @@ class IdentityController(BaseApplication): self.token_api = token.Manager(options=options) self.options = options + def noop(self, context, *args, **kw): + return '' + def authenticate(self, context, **kwargs): user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( context, **kwargs) @@ -192,12 +195,9 @@ class Router(wsgi.Router): self.token_controller = TokenController(options) mapper = self._build_map(URLMAP) - mapper.connect('/', controller=self, action='noop') + mapper.connect('/', controller=self.identity_controller, action='noop') super(Router, self).__init__(mapper) - def noop(self, context, *args, **kw): - return '' - def _build_map(self, urlmap): """Build a routes.Mapper based on URLMAP.""" mapper = routes.Mapper() diff --git a/tools/pip-requires b/tools/pip-requires index 45cdf4523f..80b03695b0 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -4,3 +4,4 @@ eventlet==0.9.12 PasteDeploy paste routes +pycli From 20bebd9f6056f8b3c23140c9297eee2567f6dc73 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 13:48:46 -0800 Subject: [PATCH 078/334] adjust default port --- etc/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/default.conf b/etc/default.conf index 176836dd1e..950edb5c1d 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -2,7 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -public_port = 5000 +public_port = 35357 admin_port = 5001 admin_token = ADMIN From aaf76955182da7196c9301300ea825e0e3403b11 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 13:49:27 -0800 Subject: [PATCH 079/334] handle unscoped requests --- keystonelight/backends/kvs.py | 6 +++++- keystonelight/keystone_compat.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 763d65fe34..89d589c91c 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -34,7 +34,10 @@ class KvsIdentity(object): raise AssertionError('Invalid tenant') tenant_ref = self.get_tenant(tenant_id) - extras_ref = self.get_extras(user_id, tenant_id) + if tenant_ref: + extras_ref = self.get_extras(user_id, tenant_id) + else: + extras_ref = {} return (user_ref, tenant_ref, extras_ref) def get_tenant(self, tenant_id): @@ -57,6 +60,7 @@ class KvsIdentity(object): return self.db.get('extras-%s-%s' % (tenant_id, user_id)) def create_user(self, id, user): + print user self.db.set('user-%s' % id, user) self.db.set('user_name-%s' % user['name'], user) return user diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 435f26efd9..6479f5e16e 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -99,11 +99,14 @@ class KeystoneController(service.BaseApplication): user=user_ref, tenant=tenant_ref, extras=extras_ref)) - catalog_ref = self.catalog_api.get_catalog( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id'], - extras=extras_ref) + if tenant_ref: + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + extras=extras_ref) + else: + catalog_ref = {} elif 'token' in auth: token = auth['token'].get('id', None) @@ -187,12 +190,13 @@ class KeystoneController(service.BaseApplication): }, 'user': {'id': user_ref['id'], 'name': user_ref['name'], - 'roles': extras_ref['roles'] or [], - 'roles_links': extras_ref['roles_links'] or [] + 'roles': extras_ref.get('roles', []), + 'roles_links': extras_ref.get('roles_links', + []) } } } - if 'tenant' in token_ref: + if 'tenant' in token_ref and token_ref['tenant']: o['access']['token']['tenant'] = token_ref['tenant'] return o From 3dac7734c6d7b8504816b77c894320d05b9a8b41 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 14:05:07 -0800 Subject: [PATCH 080/334] fix tests --- keystonelight/service.py | 4 ++-- tests/test_backend_kvs.py | 2 +- tests/test_identity_api.py | 2 +- tests/test_novaclient_compat.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/keystonelight/service.py b/keystonelight/service.py index 0294ba292c..c4b66f6d2d 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -133,7 +133,7 @@ class IdentityController(BaseApplication): return self.identity_api.get_user(context, user_id=user_id) def create_user(self, context, **kw): - user_id = kw.get('id', uuid.uuid4().hex) + user_id = kw.get('id') and kw.get('id') or uuid.uuid4().hex kw['id'] = user_id return self.identity_api.create_user(context, user_id=user_id, data=kw) @@ -153,7 +153,7 @@ class IdentityController(BaseApplication): context, tenant_name=tenant_name) def create_tenant(self, context, **kw): - tenant_id = kw.get('id', uuid.uuid4().hex) + tenant_id = kw.get('id') and kw.get('id') or uuid.uuid4().hex kw['id'] = tenant_id return self.identity_api.create_tenant( context, tenant_id=tenant_id, data=kw) diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 676b340d2b..166bc6c6d4 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -53,7 +53,7 @@ class KvsIdentity(test.TestCase): password=self.user_foo['password']) self.assertDictEquals(user_ref, self.user_foo) self.assert_(tenant_ref is None) - self.assert_(extras_ref is None) + self.assert_(not extras_ref) def test_authenticate(self): user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index a363574491..d5f6ce014b 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -65,7 +65,7 @@ class IdentityApi(test.TestCase): data = json.loads(resp.body) self.assertEquals(self.user_foo['id'], data['user']['id']) self.assertEquals(None, data['tenant']) - self.assertEquals(None, data['extras']) + self.assertEquals({}, data['extras']) def test_get_tenants(self): token = self._login() diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index bacd875abf..8f4d878f94 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -70,7 +70,7 @@ class NovaClientCompatMasterTestCase(CompatTestCase): # NOTE(termie): novaclient seems to care about the region more than # keystoneclient conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, - user='foo', + user='FOO', apikey='foo', projectid='BAR', region_name='RegionOne') From 58b8ca858c7e532bd318ae10c0b0eb4c6413dd82 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 15:47:25 -0800 Subject: [PATCH 081/334] mergeish dolph's port change --- etc/default.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/default.conf b/etc/default.conf index 950edb5c1d..f3b63a1a7a 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -2,8 +2,8 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -public_port = 35357 -admin_port = 5001 +public_port = 5000 +admin_port = 35357 admin_token = ADMIN [filter:debug] From 3479575623233c277af611ff8e8ab058b935192a Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 15:50:29 -0800 Subject: [PATCH 082/334] updates to make compatible with middleware --- keystonelight/backends/templated.py | 3 ++- keystonelight/keystone_compat.py | 8 +++++++- tests/test_keystone_compat.py | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index 274ba39abc..528b8411ff 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -47,7 +47,8 @@ class TemplatedCatalog(object): parts = k.split('.') region = parts[1] - service = parts[2] + # NOTE(termie): object-store insists on having a dash + service = parts[2].replace('_', '-') key = parts[3] region_ref = o.get(region, {}) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 6479f5e16e..8afd41c4f3 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -185,18 +185,22 @@ class KeystoneController(service.BaseApplication): def _format_token(self, token_ref): user_ref = token_ref['user'] extras_ref = token_ref['extras'] + roles = extras_ref.get('roles', []) + roles_ref = [{'id': 1, 'name': x} for x in roles] o = {'access': {'token': {'id': token_ref['id'], 'expires': token_ref['expires'] }, 'user': {'id': user_ref['id'], 'name': user_ref['name'], - 'roles': extras_ref.get('roles', []), + 'username': user_ref['name'], + 'roles': roles_ref, 'roles_links': extras_ref.get('roles_links', []) } } } if 'tenant' in token_ref and token_ref['tenant']: + token_ref['tenant']['enabled'] = True o['access']['token']['tenant'] = token_ref['tenant'] return o @@ -250,6 +254,8 @@ class KeystoneController(service.BaseApplication): return services.values() def _format_tenants_for_token(self, tenant_refs): + for x in tenant_refs: + x['enabled'] = True o = {'tenants': tenant_refs, 'tenants_links': []} return o diff --git a/tests/test_keystone_compat.py b/tests/test_keystone_compat.py index c8768c02c7..98c2ffdc5a 100644 --- a/tests/test_keystone_compat.py +++ b/tests/test_keystone_compat.py @@ -150,6 +150,7 @@ class DiabloCompatTestCase(CompatTestCase): # data['access']['serviceCatalog']) def test_validate_token_scoped(self): + raise exc.SkipTest('The docs conflict with regular usage.') client = self.client(self.app, token=self.admin_token) resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) data = json.loads(resp.body) From b0733cab49f118d008f6951bb542f69c1f0f616a Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 15 Nov 2011 16:13:13 -0800 Subject: [PATCH 083/334] change array syntax --- bin/ksl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/ksl b/bin/ksl index 3354288a68..0a9d7736f7 100755 --- a/bin/ksl +++ b/bin/ksl @@ -33,11 +33,11 @@ class BaseApp(cli.log.LoggingApp): for x in args: key, value = x.split('=', 1) # make lists if there are multiple values - if key in kv: - if type(kv) is type(tuple()): - kv[key] = kv[key] + (value,) - else: - kv[key] = (kv[key], value) + if key.endswith('[]'): + key = key[:-2] + existing = kv.get(key, []) + existing.append(value) + kv[key] = existing else: kv[key] = value return kv From 834301a731782faee761d910ea284684651626eb Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 10:58:35 -0800 Subject: [PATCH 084/334] re-indent --- keystonelight/middleware.py | 104 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/keystonelight/middleware.py b/keystonelight/middleware.py index 29e655bd9f..b8baabc98e 100644 --- a/keystonelight/middleware.py +++ b/keystonelight/middleware.py @@ -16,79 +16,79 @@ PARAMS_ENV = 'openstack.params' class TokenAuthMiddleware(wsgi.Middleware): - def process_request(self, request): - token = request.headers.get(AUTH_TOKEN_HEADER) - context = request.environ.get(CONTEXT_ENV, {}) - context['token_id'] = token - request.environ[CONTEXT_ENV] = context + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['token_id'] = token + request.environ[CONTEXT_ENV] = context class AdminTokenAuthMiddleware(wsgi.Middleware): - """A trivial filter that checks for a pre-defined admin token. + """A trivial filter that checks for a pre-defined admin token. - Sets 'is_admin' to true in the context, expected to be checked by - methods that are admin-only. + Sets 'is_admin' to true in the context, expected to be checked by + methods that are admin-only. - """ + """ - def process_request(self, request): - token = request.headers.get(AUTH_TOKEN_HEADER) - context = request.environ.get(CONTEXT_ENV, {}) - context['is_admin'] = (token == self.options['admin_token']) - request.environ[CONTEXT_ENV] = context + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['is_admin'] = (token == self.options['admin_token']) + request.environ[CONTEXT_ENV] = context class PostParamsMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as POST parameters. + """Middleware to allow method arguments to be passed as POST parameters. - Filters out the parameters `self`, `context` and anything beginning with - an underscore. + Filters out the parameters `self`, `context` and anything beginning with + an underscore. - """ + """ - def process_request(self, request): - params_parsed = request.params - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v - request.environ[PARAMS_ENV] = params + request.environ[PARAMS_ENV] = params class JsonBodyMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as serialized JSON. + """Middleware to allow method arguments to be passed as serialized JSON. - Accepting arguments as JSON is useful for accepting data that may be more - complex than simple primitives. + Accepting arguments as JSON is useful for accepting data that may be more + complex than simple primitives. - In this case we accept it as urlencoded data under the key 'json' as in - json= but this could be extended to accept raw JSON - in the POST body. + In this case we accept it as urlencoded data under the key 'json' as in + json= but this could be extended to accept raw JSON + in the POST body. - Filters out the parameters `self`, `context` and anything beginning with - an underscore. + Filters out the parameters `self`, `context` and anything beginning with + an underscore. - """ + """ - def process_request(self, request): - #if 'json' not in request.params: - # return + def process_request(self, request): + #if 'json' not in request.params: + # return - params_json = request.body - if not params_json: - return + params_json = request.body + if not params_json: + return - params_parsed = json.loads(params_json) - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v + params_parsed = json.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v - request.environ[PARAMS_ENV] = params + request.environ[PARAMS_ENV] = params From d820917da0d57fe18bb7ab85b1bf8129d25c2208 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 10:58:55 -0800 Subject: [PATCH 085/334] policty stub --- keystonelight/backends/kvs.py | 12 ++++++++++++ keystonelight/keystone_compat.py | 2 ++ tests/default.conf | 1 + tests/keystone_compat_diablo.conf | 1 + tests/keystoneclient_compat_master.conf | 1 + 5 files changed, 17 insertions(+) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 89d589c91c..5e5e3be276 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -147,3 +147,15 @@ class KvsCatalog(object): def _create_catalog(self, user_id, tenant_id, data): self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) return data + + +class KvsPolicy(object): + def __init__(self, options, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db + + def can_haz(self, target, action, credentials): + pass diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 8afd41c4f3..83fb570005 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -7,6 +7,7 @@ import routes from keystonelight import catalog from keystonelight import identity +from keystonelight import policy from keystonelight import service from keystonelight import token from keystonelight import wsgi @@ -42,6 +43,7 @@ class KeystoneController(service.BaseApplication): self.catalog_api = catalog.Manager(options) self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) + self.policy_api = policy.Manager(options) pass def noop(self, context): diff --git a/tests/default.conf b/tests/default.conf index 6d420e45e6..41245cc380 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -2,6 +2,7 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +policy_driver = keystonelight.backends.kvs.KvsPolicy public_port = 5000 admin_port = 5001 admin_token = ADMIN diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index 505c980497..efdfa3d49c 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -2,6 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +policy_driver = keystonelight.backends.kvs.KvsPolicy public_port = 5000 admin_token = ADMIN diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index 545ff26602..f5403f62b0 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -2,6 +2,7 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +policy_driver = keystonelight.backends.kvs.KvsPolicy public_port = 5000 admin_token = ADMIN From 63943c98c6ed74d42398bda38b4ddfbc3ddd4283 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 11:40:52 -0800 Subject: [PATCH 086/334] describe and add a policy backend --- README.rst | 45 +++++++++++++++++++++++++ etc/default.conf | 1 + tests/default.conf | 2 +- tests/keystone_compat_diablo.conf | 2 +- tests/keystoneclient_compat_master.conf | 2 +- 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index c0faefecbf..3c3db36a7b 100644 --- a/README.rst +++ b/README.rst @@ -102,6 +102,51 @@ CRUD is treated as an extension or additional feature to the core feature set in that it is not required that a backend support it. +---------------------------------- +Approach to Authorization (Policy) +---------------------------------- + +Various components in the system require that different actions are allowed +based on whether the user is authorized to perform that action. + +For the purposes of Keystone Light there are only a couple levels of +authorization being checked for: + + * Require that the performing user is considered an admin. + * Require that the performing user matches the user being referenced. + +Other systems wishing to use the policy engine will require additional styles +of checks and will possibly write completely custom backends. Backends included +in Keystone Light are: + + +Trivial True +------------ + +Allows all actions. + + +Simple Match +------------ + +Given a list of matches to check for, simply verify that the credentials +contain the matches. For example: + + credentials = {'user_id': 'foo', 'is_admin': 1, 'roles': ['nova:netadmin']} + + # An admin only call: + policy_api.can_haz(('is_admin:1',), credentials) + + # An admin or owner call: + policy_api.can_haz(('is_admin:1', 'user_id:foo'), + credentials) + + # A netadmin call: + policy_api.can_haz(('roles:nova:netadmin',), + credentials) + + + ----------- Still To Do ----------- diff --git a/etc/default.conf b/etc/default.conf index f3b63a1a7a..95e975c101 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -2,6 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken +policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 35357 admin_token = ADMIN diff --git a/tests/default.conf b/tests/default.conf index 41245cc380..0a3898f9c2 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -2,7 +2,7 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.kvs.KvsPolicy +policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 5001 admin_token = ADMIN diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index efdfa3d49c..ba893fbc03 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -2,7 +2,7 @@ catalog_driver = keystonelight.backends.kvs.KvsCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.kvs.KvsPolicy +policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_token = ADMIN diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index f5403f62b0..262215c2b2 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -2,7 +2,7 @@ catalog_driver = keystonelight.backends.templated.TemplatedCatalog identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.kvs.KvsPolicy +policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_token = ADMIN From 860aa86e0305c8cdc4cc509e971c39003ef0a5ea Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 11:58:43 -0800 Subject: [PATCH 087/334] add the policy code --- keystonelight/backends/policy.py | 23 +++++++++++++++++++++++ keystonelight/keystone_compat.py | 11 +++++++++-- keystonelight/policy.py | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 keystonelight/backends/policy.py create mode 100644 keystonelight/policy.py diff --git a/keystonelight/backends/policy.py b/keystonelight/backends/policy.py new file mode 100644 index 0000000000..780cf0aa00 --- /dev/null +++ b/keystonelight/backends/policy.py @@ -0,0 +1,23 @@ + + +class TrivialTrue(object): + def __init__(self, options): + self.options = options + + def can_haz(self, target, credentials): + return True + + +class SimpleMatch(object): + def __init__(self, options): + self.options = options + + def can_haz(self, target, credentials): + """Check whether key-values in target are present in credentials.""" + # TODO(termie): handle ANDs, probably by providing a tuple instead of a + # string + for requirement in target: + key, match = requirement.split(':', 1) + check = credentials.get(key) + if check == match: + return True diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 83fb570005..807a10bc58 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -44,7 +44,6 @@ class KeystoneController(service.BaseApplication): self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) self.policy_api = policy.Manager(options) - pass def noop(self, context): return {} @@ -155,7 +154,15 @@ class KeystoneController(service.BaseApplication): Optionally, also ensure that it is owned by a specific tenant. """ - assert context['is_admin'] + # TODO(termie): this stuff should probably be moved to middleware + if not context['is_admin']: + user_token_ref = self.token_api.get_token(context['token_id']) + creds = user_token_ref['extras'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(('is_admin:1', 'roles:admin'), + creds) token_ref = self.token_api.get_token(context=context, token_id=token_id) diff --git a/keystonelight/policy.py b/keystonelight/policy.py new file mode 100644 index 0000000000..147c650196 --- /dev/null +++ b/keystonelight/policy.py @@ -0,0 +1,18 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# the catalog interfaces + +import uuid + +from keystonelight import utils + + +class Manager(object): + def __init__(self, options): + self.options = options + self.driver = utils.import_object(options['policy_driver'], + options=options) + + def can_haz(self, context, target, credentials): + """Check whether the given creds can perform action on target.""" + return self.driver.can_haz(target, credentials) From e5d1050da930eac9f8cec339e2dbea140303eeb2 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 12:00:42 -0800 Subject: [PATCH 088/334] make readme use code style --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3c3db36a7b..a746089cd3 100644 --- a/README.rst +++ b/README.rst @@ -130,7 +130,7 @@ Simple Match ------------ Given a list of matches to check for, simply verify that the credentials -contain the matches. For example: +contain the matches. For example:: credentials = {'user_id': 'foo', 'is_admin': 1, 'roles': ['nova:netadmin']} From 91f209712599a00979cb3aacff1f30f538972c38 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 17 Nov 2011 12:13:09 -0800 Subject: [PATCH 089/334] add an example for capability rbac --- README.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.rst b/README.rst index a746089cd3..2d70832902 100644 --- a/README.rst +++ b/README.rst @@ -146,6 +146,30 @@ contain the matches. For example:: credentials) +Credentials are generally built from the user metadata in the 'extras' part +of the Identity API. So, adding a 'role' to the user just means adding the role +to the user metadata. + + +Capability RBAC +--------------- + +(Not yet implemented.) + +Another approach to authorization can be action-based, with a mapping of roles +to which capabilities are allowed for that role. For example:: + + credentials = {'user_id': 'foo', 'is_admin': 1, 'roles': ['nova:netadmin']} + + # add a policy + policy_api.add_policy('action:nova:add_network', ('roles:nova:netadmin',)) + + policy_api.can_haz(('action:nova:add_network',), credentials) + + +In the backend this would look up the policy for 'action:nova:add_network' and +then do what is effectively a 'Simple Match' style match against the creds. + ----------- Still To Do From cad238d912338114b4ac578da26c9ff0d9002333 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 16:07:35 -0800 Subject: [PATCH 090/334] fix tenant auth tests --- tests/test_keystoneclient_compat.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 9470ae33db..f81eb7a9f6 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -65,7 +65,7 @@ class MasterCompatTestCase(CompatTestCase): # project_id='bar') # client.authenticate() - def test_authenticate_and_tenants(self): + def test_authenticate_tenant_name_and_tenants(self): from keystoneclient.v2_0 import client as ks_client port = self.server.socket_info['socket'][1] @@ -74,7 +74,21 @@ class MasterCompatTestCase(CompatTestCase): client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, username='FOO', password='foo', - project_id='bar') + tenant_name='BAR') + client.authenticate() + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + def test_authenticate_tenant_id_and_tenants(self): + from keystoneclient.v2_0 import client as ks_client + + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, + username='FOO', + password='foo', + tenant_id='bar') client.authenticate() tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) From b42859fe7305e39ae49e72b628af4c0c6bc530ef Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 16:15:21 -0800 Subject: [PATCH 091/334] update to use the correct repo for python-novaclient --- tests/test_novaclient_compat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 8f4d878f94..854a978fd6 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -9,7 +9,7 @@ from keystonelight import test from keystonelight import utils -KEYSTONECLIENT_REPO = 'git://github.com/rackspace/python-novaclient.git' +NOVACLIENT_REPO = 'git://github.com/openstack/python-novaclient.git' class CompatTestCase(test.TestCase): @@ -21,7 +21,7 @@ class NovaClientCompatMasterTestCase(CompatTestCase): def setUp(self): super(NovaClientCompatMasterTestCase, self).setUp() - revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') + revdir = test.checkout_vendor(NOVACLIENT_REPO, 'master') self.add_path(revdir) from novaclient.keystone import client as ks_client from novaclient import client as base_client From a0d06696974361b367e2599df037a868cfe079f9 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 16:17:23 -0800 Subject: [PATCH 092/334] novaclient uses password instead of apikey --- tests/test_novaclient_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 854a978fd6..7b3b27673e 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -71,7 +71,7 @@ class NovaClientCompatMasterTestCase(CompatTestCase): # keystoneclient conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, user='FOO', - apikey='foo', + password='foo', projectid='BAR', region_name='RegionOne') client = ks_client.Client(conn) From 99f81d5e0f33c1d11d321befe9f5a54db3dc8860 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 16:24:48 -0800 Subject: [PATCH 093/334] standardize spacing --- tests/test_keystoneclient_compat.py | 133 +++++++++++++--------------- 1 file changed, 63 insertions(+), 70 deletions(-) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index f81eb7a9f6..fb60541152 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -1,9 +1,3 @@ -import copy -import json -import os -import sys - -from keystonelight import logging from keystonelight import models from keystonelight import test from keystonelight import utils @@ -12,83 +6,82 @@ from keystonelight import utils KEYSTONECLIENT_REPO = 'git://github.com/4P/python-keystoneclient.git' - class CompatTestCase(test.TestCase): - def setUp(self): - super(CompatTestCase, self).setUp() + def setUp(self): + super(CompatTestCase, self).setUp() class MasterCompatTestCase(CompatTestCase): - def setUp(self): - super(MasterCompatTestCase, self).setUp() + def setUp(self): + super(MasterCompatTestCase, self).setUp() - revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') - self.add_path(revdir) - from keystoneclient.v2_0 import client as ks_client - reload(ks_client) + revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') + self.add_path(revdir) + from keystoneclient.v2_0 import client as ks_client + reload(ks_client) - self.app = self.loadapp('keystoneclient_compat_master') - self.options = self.appconfig('keystoneclient_compat_master') + self.app = self.loadapp('keystoneclient_compat_master') + self.options = self.appconfig('keystoneclient_compat_master') - self.identity_backend = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_backend = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_backend = utils.import_object( - self.options['catalog_driver'], options=self.options) + self.identity_backend = utils.import_object( + self.options['identity_driver'], options=self.options) + self.token_backend = utils.import_object( + self.options['token_driver'], options=self.options) + self.catalog_backend = utils.import_object( + self.options['catalog_driver'], options=self.options) - self.server = self.serveapp('keystoneclient_compat_master') + self.server = self.serveapp('keystoneclient_compat_master') - self.tenant_bar = self.identity_backend.create_tenant( - 'bar', - models.Tenant(id='bar', name='BAR')) + self.tenant_bar = self.identity_backend.create_tenant( + 'bar', + models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_backend.create_user( - 'foo', - models.User(id='foo', - name='FOO', - tenants=[self.tenant_bar['id']], - password='foo')) + self.user_foo = self.identity_backend.create_user( + 'foo', + models.User(id='foo', + name='FOO', + tenants=[self.tenant_bar['id']], + password='foo')) - self.extras_bar_foo = self.identity_backend.create_extras( - self.user_foo['id'], self.tenant_bar['id'], - dict(roles=[], - roles_links=[])) + self.extras_bar_foo = self.identity_backend.create_extras( + self.user_foo['id'], self.tenant_bar['id'], + dict(roles=[], + roles_links=[])) - #def test_authenticate(self): - # from keystoneclient.v2_0 import client as ks_client + # def test_authenticate(self): + # from keystoneclient.v2_0 import client as ks_client + # + # port = self.server.socket_info['socket'][1] + # client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, + # username='foo', + # password='foo', + # project_id='bar') + # client.authenticate() - # port = self.server.socket_info['socket'][1] - # client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, - # username='foo', - # password='foo', - # project_id='bar') - # client.authenticate() + def test_authenticate_tenant_name_and_tenants(self): + from keystoneclient.v2_0 import client as ks_client - def test_authenticate_tenant_name_and_tenants(self): - from keystoneclient.v2_0 import client as ks_client + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, + username='FOO', + password='foo', + tenant_name='BAR') + client.authenticate() + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='FOO', - password='foo', - tenant_name='BAR') - client.authenticate() - tenants = client.tenants.list() - self.assertEquals(tenants[0].id, self.tenant_bar['id']) + def test_authenticate_tenant_id_and_tenants(self): + from keystoneclient.v2_0 import client as ks_client - def test_authenticate_tenant_id_and_tenants(self): - from keystoneclient.v2_0 import client as ks_client - - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='FOO', - password='foo', - tenant_id='bar') - client.authenticate() - tenants = client.tenants.list() - self.assertEquals(tenants[0].id, self.tenant_bar['id']) + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, + username='FOO', + password='foo', + tenant_id='bar') + client.authenticate() + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) From e4428dc3399406416c5b8a76ade1eb694fc19538 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 20:52:10 -0800 Subject: [PATCH 094/334] working on a tenant_create test --- tests/test_keystoneclient_compat.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index fb60541152..760f78b1f1 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -85,3 +85,20 @@ class MasterCompatTestCase(CompatTestCase): client.authenticate() tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + # FIXME(ja): this test should require the "keystone:admin" roled + # (probably the role set via --keystone_admin_role flag) + # FIXME(ja): add a test that admin endpoint is only sent to admin user + # FIXME(ja): add a test that admin endpoint returns unauthorized if not admin + def test_tenant_create(self): + from keystoneclient.v2_0 import client as ks_client + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, + username='FOO', + password='foo', + tenant_id='bar') + client.authenticate() + client.tenants.create("hello", description="My new tenant!", enabled=True) + # FIXME(ja): assert tenant was created From 5e4a877df20884ccd825e4e93cf958ed63367bf5 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 21 Dec 2011 23:20:25 -0800 Subject: [PATCH 095/334] updating of docs --- README.rst | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 2d70832902..37afb9e425 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,8 @@ -Keystone Light -============== +Keystone +======== -*Always Smooth* - -Keystone Light is a re-interpretation of the Keystone project that provides -Identity, Token and Catalog services for use specifically by projects in the -OpenStack family. +Keystone is an OpenStack project that provides Identity, Token and Catalog +services for use specifically by projects in the OpenStack family. Much of the design is precipitated from the expectation that the auth backends for most deployments will actually be shims in front of existing user systems. @@ -15,9 +12,9 @@ for most deployments will actually be shims in front of existing user systems. Data Model ---------- -Keystone Light was designed from the ground up to be amenable to multiple -styles of backends and as such many of the methods and data types will happily -accept more data than they know what to do with and pass them on to a backend. +Keystone was designed from the ground up to be amenable to multiple styles of +backends and as such many of the methods and data types will happily accept +more data than they know what to do with and pass them on to a backend. There are a few main data types: From 29e13366ea2b16624be6213bedd3d0cca4063374 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 22 Dec 2011 13:09:43 -0800 Subject: [PATCH 096/334] common ks client creation --- README.rst | 1 + tests/test_keystoneclient_compat.py | 58 ++++++++++++++--------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 37afb9e425..8397e02743 100644 --- a/README.rst +++ b/README.rst @@ -179,3 +179,4 @@ Still To Do * Keystone import. * (./) Admin-only interface * Don't check git checkouts as often, to speed up tests + * common config - http://wiki.openstack.org/CommonConfigModule \ No newline at end of file diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 760f78b1f1..9ad01b1770 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -3,13 +3,28 @@ from keystonelight import test from keystonelight import utils -KEYSTONECLIENT_REPO = 'git://github.com/4P/python-keystoneclient.git' +KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' class CompatTestCase(test.TestCase): def setUp(self): super(CompatTestCase, self).setUp() + def _url(self): + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + return "http://localhost:%s/v2.0/" % port + + def _client(self, **kwargs): + from keystoneclient.v2_0 import client as ks_client + + port = self.server.socket_info['socket'][1] + self.options['public_port'] = port + kc = ks_client.Client(**kwargs) + kc.authenticate() + return kc + class MasterCompatTestCase(CompatTestCase): def setUp(self): @@ -59,30 +74,18 @@ class MasterCompatTestCase(CompatTestCase): # client.authenticate() def test_authenticate_tenant_name_and_tenants(self): - from keystoneclient.v2_0 import client as ks_client - - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='FOO', - password='foo', - tenant_name='BAR') - client.authenticate() + client = self._client(auth_url=self._url(), + username='FOO', + password='foo', + tenant_name='BAR') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_tenant_id_and_tenants(self): - from keystoneclient.v2_0 import client as ks_client - - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='FOO', - password='foo', - tenant_id='bar') - client.authenticate() + client = self._client(auth_url=self._url(), + username='FOO', + password='foo', + tenant_id='bar') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) @@ -91,14 +94,9 @@ class MasterCompatTestCase(CompatTestCase): # FIXME(ja): add a test that admin endpoint is only sent to admin user # FIXME(ja): add a test that admin endpoint returns unauthorized if not admin def test_tenant_create(self): - from keystoneclient.v2_0 import client as ks_client - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - client = ks_client.Client(auth_url="http://localhost:%s/v2.0/" % port, - username='FOO', - password='foo', - tenant_id='bar') - client.authenticate() + client = self._client(auth_url=self._url(), + username='FOO', + password='foo', + tenant_name='BAR') client.tenants.create("hello", description="My new tenant!", enabled=True) # FIXME(ja): assert tenant was created From 82f6445a2482a2f581f6aa3bb8c9ca6ade8b13a7 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 22 Dec 2011 15:20:55 -0800 Subject: [PATCH 097/334] make create_tenant work for keystone api --- keystonelight/backends/policy.py | 3 ++- keystonelight/keystone_compat.py | 33 +++++++++++++++++++++++++++-- tests/test_keystoneclient_compat.py | 1 + 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/keystonelight/backends/policy.py b/keystonelight/backends/policy.py index 780cf0aa00..d6bb78374d 100644 --- a/keystonelight/backends/policy.py +++ b/keystonelight/backends/policy.py @@ -1,4 +1,4 @@ - +import logging class TrivialTrue(object): def __init__(self, options): @@ -21,3 +21,4 @@ class SimpleMatch(object): check = credentials.get(key) if check == match: return True + diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 807a10bc58..85f947faf6 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -2,6 +2,7 @@ # this is the web service frontend that emulates keystone import logging +import uuid import routes @@ -34,6 +35,10 @@ class KeystoneRouter(wsgi.Router): controller=self.keystone_controller, action='tenants_for_token', conditions=dict(method=['GET'])) + mapper.connect('/tenants', + controller=self.keystone_controller, + action='create_tenant', + conditions=dict(method=['POST'])) super(KeystoneRouter, self).__init__(mapper) @@ -156,12 +161,14 @@ class KeystoneController(service.BaseApplication): """ # TODO(termie): this stuff should probably be moved to middleware if not context['is_admin']: - user_token_ref = self.token_api.get_token(context['token_id']) + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) creds = user_token_ref['extras'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') # Accept either is_admin or the admin role - assert self.policy_api.can_haz(('is_admin:1', 'roles:admin'), + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), creds) token_ref = self.token_api.get_token(context=context, @@ -191,6 +198,28 @@ class KeystoneController(service.BaseApplication): tenant_id=tenant_id)) return self._format_tenants_for_token(tenant_refs) + def create_tenant(self, context, **kw): + # TODO(termie): this stuff should probably be moved to middleware + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['extras'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + tenant_ref = kw.get('tenant') + tenant_id = (tenant_ref.get('id') + and tenant_ref.get('id') + or uuid.uuid4().hex) + tenant_ref['id'] = tenant_id + + tenant = self.identity_api.create_tenant( + context, tenant_id=tenant_id, data=tenant_ref) + return {'tenant': tenant} + def _format_token(self, token_ref): user_ref = token_ref['user'] extras_ref = token_ref['extras'] diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 9ad01b1770..1c64ddb3d6 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -61,6 +61,7 @@ class MasterCompatTestCase(CompatTestCase): self.extras_bar_foo = self.identity_backend.create_extras( self.user_foo['id'], self.tenant_bar['id'], dict(roles=[], + is_admin='1', roles_links=[])) # def test_authenticate(self): From 5ff67d74703e2ff2a42cafe8df4b4084f82eb19d Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 22 Dec 2011 15:22:19 -0800 Subject: [PATCH 098/334] whitespace --- keystonelight/backends/policy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keystonelight/backends/policy.py b/keystonelight/backends/policy.py index d6bb78374d..d2c339172e 100644 --- a/keystonelight/backends/policy.py +++ b/keystonelight/backends/policy.py @@ -1,5 +1,6 @@ import logging + class TrivialTrue(object): def __init__(self, options): self.options = options From 26a4cde397c16452ec0d3f70b02c3e63a3a58736 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 22 Dec 2011 15:22:29 -0800 Subject: [PATCH 099/334] whitespace --- keystonelight/backends/policy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/keystonelight/backends/policy.py b/keystonelight/backends/policy.py index d2c339172e..6bdcacb80b 100644 --- a/keystonelight/backends/policy.py +++ b/keystonelight/backends/policy.py @@ -22,4 +22,3 @@ class SimpleMatch(object): check = credentials.get(key) if check == match: return True - From b4eba62b214a4dbc0c87e4c8e57c6643a5a0c9d7 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 27 Dec 2011 12:10:39 -0800 Subject: [PATCH 100/334] novaclient now requires prettytable --- tools/pip-requires-test | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 257c0f7e57..25b20295f9 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -12,3 +12,6 @@ nose # for python-keystoneclient httplib2 pep8 + +# for python-novaclient +prettytable From 7541ed49954dea081ecb3aa97246b42b4f3c92c3 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 27 Dec 2011 11:20:55 -0800 Subject: [PATCH 101/334] documentation driven development --- README.rst | 58 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 8397e02743..2c9e5534ce 100644 --- a/README.rst +++ b/README.rst @@ -1,27 +1,75 @@ Keystone ======== -Keystone is an OpenStack project that provides Identity, Token and Catalog -services for use specifically by projects in the OpenStack family. +Keystone is an OpenStack project that provides Identity, Token, Catalog and +Policy services for use specifically by projects in the OpenStack family. Much of the design is precipitated from the expectation that the auth backends for most deployments will actually be shims in front of existing user systems. +------------ +The Services +------------ + +Keystone is organized as a group of services exposed on one or many endpoints. +Many of these services are used in a combined fashion by the frontend, for +example an authenticate call will validate user/tenant credentials with the +Identity service and, upon success, create and return a token with the Token +service. + + +Identity +-------- + +The Identity service provides auth credential validation and data about Users, +Tenants and Roles, as well as any associated metadata. + +In the basic case all this data is managed by the service, allowing the service +to manage all the CRUD associated with the data. + +In other cases, this data is pulled, by varying degrees, from an authoritative +backend service. An example of this would be when backending on LDAP. See +`LDAP Backend` below for more details. + + +Token +----- + +The Token service validates and manages Tokens used for authenticating requests +once a user/tenant's credentials have already been verified. + + +Catalog +------- + +The Catalog service provides an endpoint registry used for endpoint discovery. + + +Policy +------ + +The Policy service provides a rule-based authorization engine and the +associated rule management interface. + + + ---------- Data Model ---------- Keystone was designed from the ground up to be amenable to multiple styles of -backends and as such many of the methods and data types will happily accept +backends and as such many of the methods and data types will happily accept more data than they know what to do with and pass them on to a backend. There are a few main data types: * **User**: has account credentials, is associated with one or more tenants * **Tenant**: unit of ownership in openstack, contains one or more users + * **Role**: a first-class piece of metadata associated with many user-tenant pairs. * **Token**: identifying credential associated with a user or user and tenant - * **Extras**: bucket of key-values associated with a user-tenant pair, typically used to define roles. + * **Extras**: bucket of key-value metadata associated with a user-tenant pair. + * **Rule**: describes a set of requirements for performing an action. While the general data model allows a many-to-many relationship between Users and Tenants and a many-to-one relationship between Extras and User-Tenant pairs, @@ -179,4 +227,4 @@ Still To Do * Keystone import. * (./) Admin-only interface * Don't check git checkouts as often, to speed up tests - * common config - http://wiki.openstack.org/CommonConfigModule \ No newline at end of file + * common config - http://wiki.openstack.org/CommonConfigModule From 8425eabe6ffb9dcdfe205d559ca33e5cbf749428 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 27 Dec 2011 12:06:14 -0800 Subject: [PATCH 102/334] add basic fixture functionality --- keystonelight/models.py | 10 ++++++++++ keystonelight/test.py | 31 +++++++++++++++++++++++++++++++ tests/default_fixtures.py | 16 ++++++++++++++++ tests/test_backend_kvs.py | 17 ++--------------- 4 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 tests/default_fixtures.py diff --git a/keystonelight/models.py b/keystonelight/models.py index 08747cddb5..a3028d467f 100644 --- a/keystonelight/models.py +++ b/keystonelight/models.py @@ -15,3 +15,13 @@ class User(dict): class Tenant(dict): def __init__(self, id=None, *args, **kw): super(Tenant, self).__init__(id=id, *args, **kw) + + +class Role(dict): + def __init__(self, id=None, *args, **kw): + super(Role, self).__init__(id=id, *args, **kw) + + +class Extras(dict): + def __init__(self, user=None, tenant=None, *args, **kw): + super(Extras, self).__init__(user=User, tenant=tenant, *args, **kw) diff --git a/keystonelight/test.py b/keystonelight/test.py index 72cc63ca55..5e420e3e98 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -8,6 +8,7 @@ import sys from paste import deploy from keystonelight import logging +from keystonelight import models from keystonelight import utils from keystonelight import wsgi @@ -84,6 +85,36 @@ class TestCase(unittest.TestCase): sys.path.remove(path) super(TestCase, self).tearDown() + def load_fixtures(self, fixtures): + """Really quite basic and naive fixture loading based on a python module. + + Expects that the various APIs into the various services are already + defined on `self`. + + """ + # TODO(termie): doing something from json, probably based on Django's + # loaddata will be much preferred. + for tenant in fixtures.TENANTS: + rv = self.identity_api.create_tenant( + tenant['id'], models.Tenant(**tenant)) + setattr(self, 'tenant_%s' % tenant['id'], rv) + + for user in fixtures.USERS: + rv = self.identity_api.create_user(user['id'], models.User(**user)) + setattr(self, 'user_%s' % user['id'], rv) + + for role in fixtures.ROLES: + rv = self.identity_api.create_role(role['id'], models.Role(**role)) + setattr(self, 'role_%s' % role['id'], rv) + + for extras in fixtures.EXTRAS: + extras_ref = extras.copy() + del extras_ref['user'] + del extras_ref['tenant'] + rv = self.identity_api.create_extras( + extras['user'], extras['tenant'], models.Extras(**extras_ref)) + setattr(self, 'extras_%s%s' % (extras['user'], extras['tenant']), rv) + def loadapp(self, config): if not config.startswith('config:'): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py new file mode 100644 index 0000000000..15a8b60a34 --- /dev/null +++ b/tests/default_fixtures.py @@ -0,0 +1,16 @@ +TENANTS = [ + {'id': 'bar', 'name': 'BAR'}, + ] + +USERS = [ + {'id': 'foo', 'name': 'FOO', 'password': 'foo2', 'tenants': ['bar',]}, + ] + +EXTRAS = [ + {'user': 'foo', 'tenant': 'bar', 'extra': 'extra'}, + ] + +#ROLES = [ +# {'id': 'keystone_admin', 'name': 'Keystone Admin'}, +# ] +ROLES = [] diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 166bc6c6d4..df88733d42 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -4,27 +4,14 @@ from keystonelight import models from keystonelight import test from keystonelight.backends import kvs +import default_fixtures class KvsIdentity(test.TestCase): def setUp(self): super(KvsIdentity, self).setUp() options = self.appconfig('default') self.identity_api = kvs.KvsIdentity(options=options, db={}) - self._load_fixtures() - - def _load_fixtures(self): - self.tenant_bar = self.identity_api.create_tenant( - 'bar', - models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_api.create_user( - 'foo', - models.User(id='foo', - name='FOO', - password='foo2', - tenants=[self.tenant_bar['id']])) - self.extras_foobar = self.identity_api.create_extras( - 'foo', 'bar', - {'extra': 'extra'}) + self.load_fixtures(default_fixtures) def test_authenticate_bad_user(self): self.assertRaises(AssertionError, From a32c73c53504ed6a13ca1eba215320d04ffe48cc Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 27 Dec 2011 12:50:11 -0800 Subject: [PATCH 103/334] speed up tests --- .gitignore | 3 +++ run_tests.sh | 2 +- tests/{test_keystone_compat.py => test_legacy_compat.py} | 0 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .gitignore rename tests/{test_keystone_compat.py => test_legacy_compat.py} (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..5844516e6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +vendor +.ksl-venv +run_tests.log diff --git a/run_tests.sh b/run_tests.sh index 621a6efa50..02c4458330 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -59,7 +59,7 @@ done # If enabled, tell nose to collect coverage data if [ $coverage -eq 1 ]; then - noseopts="$noseopts --with-coverage --cover-package=nova" + noseopts="$noseopts --with-coverage --cover-package=keystone" fi function run_tests { diff --git a/tests/test_keystone_compat.py b/tests/test_legacy_compat.py similarity index 100% rename from tests/test_keystone_compat.py rename to tests/test_legacy_compat.py From 32aa1dedb7cd5f1cae6531e72b122fca60ec9e19 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 27 Dec 2011 12:51:23 -0800 Subject: [PATCH 104/334] add role crud --- keystonelight/backends/kvs.py | 17 ++++++++++++++++- keystonelight/identity.py | 12 ++++++++++++ keystonelight/test.py | 11 +++++++++++ tests/default_fixtures.py | 7 +++---- tests/test_backend_kvs.py | 5 +++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 5e5e3be276..37b1aa0282 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -59,8 +59,11 @@ class KvsIdentity(object): def get_extras(self, user_id, tenant_id): return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + def get_role(self, role_id): + role_ref = self.db.get('role-%s' % role_id) + return role_ref + def create_user(self, id, user): - print user self.db.set('user-%s' % id, user) self.db.set('user_name-%s' % user['name'], user) return user @@ -110,6 +113,18 @@ class KvsIdentity(object): self.db.delete('extras-%s-%s' % (tenant_id, user_id)) return None + def create_role(self, id, role): + self.db.set('role-%s' % id, role) + return role + + def update_role(self, id, role): + self.db.set('role-%s' % id, role) + return role + + def delete_role(self, id): + self.db.delete('role-%s' % id) + return None + class KvsToken(object): def __init__(self, options, db=None): diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 42c73e0543..47d5403df6 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -34,6 +34,9 @@ class Manager(object): def get_extras(self, context, user_id, tenant_id): return self.driver.get_extras(user_id, tenant_id) + def get_role(self, context, role_id): + return self.driver.get_role(role_id) + # CRUD operations def create_user(self, context, user_id, data): return self.driver.create_user(user_id, data) @@ -61,3 +64,12 @@ class Manager(object): def delete_extras(self, context, user_id, tenant_id): return self.driver.delete_extras(user_id, tenant_id) + + def create_role(self, context, role_id, data): + return self.driver.create_role(role_id, data) + + def update_role(self, context, role_id, data): + return self.driver.update_role(role_id, data) + + def delete_role(self, context, role_id): + return self.driver.delete_role(role_id) diff --git a/keystonelight/test.py b/keystonelight/test.py index 5e420e3e98..bcfee343a2 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -4,6 +4,7 @@ import os import unittest import subprocess import sys +import time from paste import deploy @@ -27,13 +28,23 @@ def checkout_vendor(repo, rev): name = name[:-4] revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) + modcheck = os.path.join(VENDOR, '.%s-%s' % (name, rev.replace('/', '_'))) try: + if os.path.exists(modcheck): + mtime = os.stat(modcheck).st_mtime + if int(time.time()) - mtime < 1000: + return revdir + if not os.path.exists(revdir): utils.git('clone', repo, revdir) cd(revdir) utils.git('pull') utils.git('checkout', '-q', rev) + + # write out a modified time + with open(modcheck, 'w') as fd: + fd.write('1') except subprocess.CalledProcessError as e: logging.warning('Failed to checkout %s', repo) pass diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 15a8b60a34..8a6d6f3a77 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -10,7 +10,6 @@ EXTRAS = [ {'user': 'foo', 'tenant': 'bar', 'extra': 'extra'}, ] -#ROLES = [ -# {'id': 'keystone_admin', 'name': 'Keystone Admin'}, -# ] -ROLES = [] +ROLES = [ + {'id': 'keystone_admin', 'name': 'Keystone Admin'}, + ] diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index df88733d42..8ec1e60720 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -97,6 +97,11 @@ class KvsIdentity(test.TestCase): tenant_id=self.tenant_bar['id']) self.assertDictEquals(extras_ref, self.extras_foobar) + def test_get_role(self): + role_ref = self.identity_api.get_role( + role_id=self.role_keystone_admin['id']) + self.assertDictEquals(role_ref, self.role_keystone_admin) + class KvsToken(test.TestCase): def setUp(self): From 2e1558ebd2dba1d2adb89160d1bb1f4e2020f762 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 28 Dec 2011 13:58:49 -0800 Subject: [PATCH 105/334] clean up keystoneclient setup --- keystonelight/keystone_compat.py | 17 +++++++---- keystonelight/test.py | 16 ++++++++++- tests/test_backend_kvs.py | 4 +-- tests/test_keystoneclient_compat.py | 44 +++++++++++------------------ 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 85f947faf6..0055955ee0 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -150,7 +150,14 @@ class KeystoneController(service.BaseApplication): tenant_id=tenant_ref['id'], extras=extras_ref) - return self._format_authenticate(token_ref, catalog_ref) + # TODO(termie): optimize this call at some point and put it into the + # the return for extras + # fill out the roles in the extras + roles_ref = [] + for role_id in extras_ref.get('roles', []): + roles_ref.append(self.identity_api.get_role(context, role_id)) + + return self._format_authenticate(token_ref, roles_ref, catalog_ref) #admin-only def validate_token(self, context, token_id, belongs_to=None): @@ -220,11 +227,9 @@ class KeystoneController(service.BaseApplication): context, tenant_id=tenant_id, data=tenant_ref) return {'tenant': tenant} - def _format_token(self, token_ref): + def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] extras_ref = token_ref['extras'] - roles = extras_ref.get('roles', []) - roles_ref = [{'id': 1, 'name': x} for x in roles] o = {'access': {'token': {'id': token_ref['id'], 'expires': token_ref['expires'] }, @@ -242,8 +247,8 @@ class KeystoneController(service.BaseApplication): o['access']['token']['tenant'] = token_ref['tenant'] return o - def _format_authenticate(self, token_ref, catalog_ref): - o = self._format_token(token_ref) + def _format_authenticate(self, token_ref, roles_ref, catalog_ref): + o = self._format_token(token_ref, roles_ref) o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) return o diff --git a/keystonelight/test.py b/keystonelight/test.py index bcfee343a2..a0efb40d51 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -96,8 +96,22 @@ class TestCase(unittest.TestCase): sys.path.remove(path) super(TestCase, self).tearDown() + #TODO(termie): probably make this take an argument and use that for `options` + def load_backends(self): + """Hacky shortcut to load the backends for data manipulation. + + Expects self.options to have already been set. + + """ + self.identity_api = utils.import_object( + self.options['identity_driver'], options=self.options) + self.token_api = utils.import_object( + self.options['token_driver'], options=self.options) + self.catalog_api = utils.import_object( + self.options['catalog_driver'], options=self.options) + def load_fixtures(self, fixtures): - """Really quite basic and naive fixture loading based on a python module. + """Hacky basic and naive fixture loading based on a python module. Expects that the various APIs into the various services are already defined on `self`. diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 8ec1e60720..d23605b61b 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -9,8 +9,8 @@ import default_fixtures class KvsIdentity(test.TestCase): def setUp(self): super(KvsIdentity, self).setUp() - options = self.appconfig('default') - self.identity_api = kvs.KvsIdentity(options=options, db={}) + self.options = self.appconfig('default') + self.identity_api = kvs.KvsIdentity(options=self.options, db={}) self.load_fixtures(default_fixtures) def test_authenticate_bad_user(self): diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 1c64ddb3d6..e0eec5f457 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -2,6 +2,8 @@ from keystonelight import models from keystonelight import test from keystonelight import utils +import default_fixtures + KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' @@ -37,32 +39,18 @@ class MasterCompatTestCase(CompatTestCase): self.app = self.loadapp('keystoneclient_compat_master') self.options = self.appconfig('keystoneclient_compat_master') - - self.identity_backend = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_backend = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_backend = utils.import_object( - self.options['catalog_driver'], options=self.options) + self.load_backends() + self.load_fixtures(default_fixtures) self.server = self.serveapp('keystoneclient_compat_master') - self.tenant_bar = self.identity_backend.create_tenant( - 'bar', - models.Tenant(id='bar', name='BAR')) - - self.user_foo = self.identity_backend.create_user( - 'foo', - models.User(id='foo', - name='FOO', - tenants=[self.tenant_bar['id']], - password='foo')) - - self.extras_bar_foo = self.identity_backend.create_extras( + # TODO(termie): is_admin is being deprecated once the policy stuff + # is all working + # TODO(termie): add an admin user to the fixtures and use that user + # override the fixtures, for now + self.extras_foobar = self.identity_api.update_extras( self.user_foo['id'], self.tenant_bar['id'], - dict(roles=[], - is_admin='1', - roles_links=[])) + dict(roles=['keystone_admin'], is_admin='1')) # def test_authenticate(self): # from keystoneclient.v2_0 import client as ks_client @@ -77,7 +65,7 @@ class MasterCompatTestCase(CompatTestCase): def test_authenticate_tenant_name_and_tenants(self): client = self._client(auth_url=self._url(), username='FOO', - password='foo', + password='foo2', tenant_name='BAR') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) @@ -85,7 +73,7 @@ class MasterCompatTestCase(CompatTestCase): def test_authenticate_tenant_id_and_tenants(self): client = self._client(auth_url=self._url(), username='FOO', - password='foo', + password='foo2', tenant_id='bar') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) @@ -93,11 +81,13 @@ class MasterCompatTestCase(CompatTestCase): # FIXME(ja): this test should require the "keystone:admin" roled # (probably the role set via --keystone_admin_role flag) # FIXME(ja): add a test that admin endpoint is only sent to admin user - # FIXME(ja): add a test that admin endpoint returns unauthorized if not admin + # FIXME(ja): add a test that admin endpoint returns unauthorized if not + # admin def test_tenant_create(self): client = self._client(auth_url=self._url(), username='FOO', - password='foo', + password='foo2', tenant_name='BAR') - client.tenants.create("hello", description="My new tenant!", enabled=True) + client.tenants.create( + "hello", description="My new tenant!", enabled=True) # FIXME(ja): assert tenant was created From 9e8ec252aa8a7704c4600b2ab367efff6a13902f Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 28 Dec 2011 14:16:29 -0800 Subject: [PATCH 106/334] clean up test_identity_api --- keystonelight/models.py | 2 +- keystonelight/test.py | 2 ++ tests/test_identity_api.py | 28 +++++----------------------- 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/keystonelight/models.py b/keystonelight/models.py index a3028d467f..5cf76f6fea 100644 --- a/keystonelight/models.py +++ b/keystonelight/models.py @@ -24,4 +24,4 @@ class Role(dict): class Extras(dict): def __init__(self, user=None, tenant=None, *args, **kw): - super(Extras, self).__init__(user=User, tenant=tenant, *args, **kw) + super(Extras, self).__init__(user=user, tenant=tenant, *args, **kw) diff --git a/keystonelight/test.py b/keystonelight/test.py index a0efb40d51..282ad2d300 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -134,6 +134,8 @@ class TestCase(unittest.TestCase): for extras in fixtures.EXTRAS: extras_ref = extras.copy() + # TODO(termie): these will probably end up in the model anyway, so this + # may be futile del extras_ref['user'] del extras_ref['tenant'] rv = self.identity_api.create_extras( diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index d5f6ce014b..b0e55a958e 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -7,35 +7,17 @@ from keystonelight import test from keystonelight import utils from keystonelight.backends import kvs +import default_fixtures + class IdentityApi(test.TestCase): def setUp(self): super(IdentityApi, self).setUp() self.options = self.appconfig('default') - app = self.loadapp('default') - self.app = app + self.app = self.loadapp('default') - self.identity_backend = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_backend = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_backend = utils.import_object( - self.options['catalog_driver'], options=self.options) - self._load_fixtures() - - def _load_fixtures(self): - self.tenant_bar = self.identity_backend.create_tenant( - 'bar', - models.Tenant(id='bar', name='BAR')) - self.user_foo = self.identity_backend.create_user( - 'foo', - models.User(id='foo', - name='FOO', - password='foo2', - tenants=[self.tenant_bar['id']])) - self.extras_foobar = self.identity_backend.create_extras( - 'foo', 'bar', - {'extra': 'extra'}) + self.load_backends() + self.load_fixtures(default_fixtures) def _login(self): c = client.TestClient(self.app) From 909770de2a317baf93f2f5a6710867a7f5c7b1fd Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 28 Dec 2011 14:20:16 -0800 Subject: [PATCH 107/334] move novaclient tests over also --- tests/test_novaclient_compat.py | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 7b3b27673e..1a441a4c1d 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -8,6 +8,8 @@ from keystonelight import models from keystonelight import test from keystonelight import utils +import default_fixtures + NOVACLIENT_REPO = 'git://github.com/openstack/python-novaclient.git' @@ -30,33 +32,10 @@ class NovaClientCompatMasterTestCase(CompatTestCase): self.app = self.loadapp('keystoneclient_compat_master') self.options = self.appconfig('keystoneclient_compat_master') - - self.identity_backend = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_backend = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_backend = utils.import_object( - self.options['catalog_driver'], options=self.options) - + self.load_backends() + self.load_fixtures(default_fixtures) self.server = self.serveapp('keystoneclient_compat_master') - self.tenant_bar = self.identity_backend.create_tenant( - 'bar', - models.Tenant(id='bar', name='BAR')) - - self.user_foo = self.identity_backend.create_user( - 'foo', - models.User(id='foo', - name='FOO', - tenants=[self.tenant_bar['id']], - password='foo')) - - self.extras_bar_foo = self.identity_backend.create_extras( - self.user_foo['id'], self.tenant_bar['id'], - dict(roles=[], - roles_links=[])) - - def test_authenticate_and_tenants(self): from novaclient.keystone import client as ks_client from novaclient import client as base_client @@ -71,7 +50,7 @@ class NovaClientCompatMasterTestCase(CompatTestCase): # keystoneclient conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, user='FOO', - password='foo', + password='foo2', projectid='BAR', region_name='RegionOne') client = ks_client.Client(conn) From 4b55fa5e8e5f45f0155879f8d8afa8caf73898af Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 29 Dec 2011 17:40:32 -0800 Subject: [PATCH 108/334] Split keystone compat by admin and service endpoints --- .gitignore | 2 + bin/keystone | 29 ++- etc/default.conf | 16 +- keystonelight/catalog.py | 2 - keystonelight/keystone_compat.py | 290 ++++++++++++++++++------ keystonelight/test.py | 13 +- keystonelight/wsgi.py | 20 +- tests/default.conf | 18 +- tests/keystone_compat_diablo.conf | 17 +- tests/keystoneclient_compat_master.conf | 23 +- tests/test_identity_api.py | 3 - tests/test_keystoneclient_compat.py | 46 ++-- 12 files changed, 347 insertions(+), 132 deletions(-) diff --git a/.gitignore b/.gitignore index 5844516e6c..540bc6e59a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.pyc +*.swp vendor .ksl-venv run_tests.log diff --git a/bin/keystone b/bin/keystone index f902c31b1d..7775b20337 100755 --- a/bin/keystone +++ b/bin/keystone @@ -1,6 +1,7 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import greenlet import logging import os import sys @@ -20,6 +21,22 @@ from paste import deploy from keystonelight import wsgi +def create_server(name, port): + app = deploy.loadapp('config:%s' % conf, name=name) + return wsgi.Server(app, port) + + +def serve(*servers): + for server in servers: + server.start() + + for server in servers: + try: + server.wait() + except greenlet.GreenletExit: + pass + + if __name__ == '__main__': default_conf = os.path.join(possible_topdir, 'etc', @@ -27,9 +44,11 @@ if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) conf = len(sys.argv) > 1 and sys.argv[1] or default_conf - app = deploy.loadapp('config:%s' % conf) options = deploy.appconfig('config:%s' % conf) - server = wsgi.Server() - server.start(app, int(options['public_port'])) - server.start(app, int(options['admin_port'])) - server.wait() + + servers = [] + servers.append(create_server('keystone_admin', + int(options['admin_port']))) + servers.append(create_server('keystonelight', + int(options['public_port']))) + serve(*servers) diff --git a/etc/default.conf b/etc/default.conf index 95e975c101..f50c7706d6 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -22,16 +22,22 @@ paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[app:keystone] -paste.app_factory = keystonelight.keystone_compat:app_factory +[app:keystone_service] +paste.app_factory = keystonelight.keystone_compat:service_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +[app:keystone_admin] +paste.app_factory = keystonelight.keystone_compat:admin_app_factory [pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_service_api] +pipeline = token_auth admin_token_auth json_body debug keystone_service + +[pipeline:keystone_admin_api] +pipeline = token_auth admin_token_auth json_body debug keystone_admin + [composite:main] use = egg:Paste#urlmap / = keystonelight_api -/v2.0 = keystone_api +/v2.0 = keystone_service_api diff --git a/keystonelight/catalog.py b/keystonelight/catalog.py index 221e1e4873..f7df4f66d4 100644 --- a/keystonelight/catalog.py +++ b/keystonelight/catalog.py @@ -2,8 +2,6 @@ # the catalog interfaces -import uuid - from keystonelight import utils diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 0055955ee0..443680b37a 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -1,9 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # this is the web service frontend that emulates keystone -import logging import uuid - import routes from keystonelight import catalog @@ -14,44 +12,123 @@ from keystonelight import token from keystonelight import wsgi -class KeystoneRouter(wsgi.Router): +class KeystoneAdminRouter(wsgi.Router): def __init__(self, options): self.options = options - self.keystone_controller = KeystoneController(options) mapper = routes.Mapper() - mapper.connect('/', - controller=self.keystone_controller, - action='noop') + + # Token Operations + auth_controller = KeystoneTokenController(self.options) mapper.connect('/tokens', - controller=self.keystone_controller, + controller=auth_controller, action='authenticate', conditions=dict(method=['POST'])) mapper.connect('/tokens/{token_id}', - controller=self.keystone_controller, + controller=auth_controller, action='validate_token', conditions=dict(method=['GET'])) + mapper.connect('/tokens/{token_id}/endpoints', + controller=auth_controller, + action='endpoints', + conditions=dict(method=['GET'])) + + # Tenant Operations + tenant_controller = KeystoneTenantController(self.options) mapper.connect('/tenants', - controller=self.keystone_controller, - action='tenants_for_token', + controller=tenant_controller, + action='get_tenants_for_token', + conditions=dict(method=['GET'])) + mapper.connect('/tenants/{tenant_id}', + controller=tenant_controller, + action='get_tenant', conditions=dict(method=['GET'])) mapper.connect('/tenants', - controller=self.keystone_controller, + controller=tenant_controller, action='create_tenant', conditions=dict(method=['POST'])) - super(KeystoneRouter, self).__init__(mapper) + + # User Operations + user_controller = KeystoneUserController(self.options) + mapper.connect('/users/{user_id}', + controller=user_controller, + action='get_user', + conditions=dict(method=['GET'])) + + # Role Operations + roles_controller = KeystoneRoleController(self.options) + mapper.connect('/tenants/{tenant_id}/users/{user_id}/roles', + controller=roles_controller, + action='get_user_roles', + conditions=dict(method=['GET'])) + mapper.connect('/users/{user_id}/roles', + controller=user_controller, + action='get_user_roles', + conditions=dict(method=['GET'])) + + # Miscellaneous Operations + version_controller = KeystoneVersionController(self.options) + mapper.connect('/', + controller=version_controller, + action='get_version_info', module='admin/version', + conditions=dict(method=['GET'])) + + extensions_controller = KeystoneExtensionsController(self.options) + mapper.connect('/extensions', + controller=extensions_controller, + action='get_extensions_info', + conditions=dict(method=['GET'])) + + super(KeystoneAdminRouter, self).__init__(mapper) -class KeystoneController(service.BaseApplication): +class KeystoneServiceRouter(wsgi.Router): + def __init__(self, options): + self.options = options + + mapper = routes.Mapper() + + # Token Operations + auth_controller = KeystoneTokenController(self.options) + mapper.connect('/tokens', + controller=auth_controller, + action='authenticate', + conditions=dict(method=['POST'])) + mapper.connect('/ec2tokens', + controller=auth_controller, + action='authenticate_ec2', + conditions=dict(method=['POST'])) + + # Tenant Operations + tenant_controller = KeystoneTenantController(self.options) + mapper.connect('/tenants', + controller=tenant_controller, + action='get_tenants_for_token', + conditions=dict(method=['GET'])) + + # Miscellaneous + version_controller = KeystoneVersionController(self.options) + mapper.connect('/', + controller=version_controller, + action='get_version_info', + module='service/version', + conditions=dict(method=['GET'])) + + extensions_controller = KeystoneExtensionsController(self.options) + mapper.connect('/extensions', + controller=extensions_controller, + action='get_extensions_info', + conditions=dict(method=['GET'])) + + super(KeystoneServiceRouter, self).__init__(mapper) + + +class KeystoneTokenController(service.BaseApplication): def __init__(self, options): self.options = options self.catalog_api = catalog.Manager(options) self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) - self.policy_api = policy.Manager(options) - - def noop(self, context): - return {} def authenticate(self, context, auth=None): """Authenticate credentials and return a token. @@ -159,7 +236,10 @@ class KeystoneController(service.BaseApplication): return self._format_authenticate(token_ref, roles_ref, catalog_ref) - #admin-only + def authenticate_ec2(self, context): + raise NotImplemented() + + # admin only def validate_token(self, context, token_id, belongs_to=None): """Check that a token is valid. @@ -184,48 +264,19 @@ class KeystoneController(service.BaseApplication): assert token_ref['tenant']['id'] == belongs_to return self._format_token(token_ref) - def tenants_for_token(self, context): - """Get valid tenants for token based on token used to authenticate. - - Pulls the token from the context, validates it and gets the valid - tenants for the user in the token. - - Doesn't care about token scopedness. - - """ + def endpoints(self, context, token_id): + """Return service catalog endpoints.""" token_ref = self.token_api.get_token(context=context, - token_id=context['token_id']) - assert token_ref is not None + token_id=token_id) + catalog_ref = self.catalog_api.get_catalog(context, + token_ref['user_id'], + token_ref['tenant_id']) + return self._format_catalog(catalog_ref) - user_ref = token_ref['user'] - tenant_refs = [] - for tenant_id in user_ref['tenants']: - tenant_refs.append(self.identity_api.get_tenant( - context=context, - tenant_id=tenant_id)) - return self._format_tenants_for_token(tenant_refs) - - def create_tenant(self, context, **kw): - # TODO(termie): this stuff should probably be moved to middleware - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['extras'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) - tenant_ref = kw.get('tenant') - tenant_id = (tenant_ref.get('id') - and tenant_ref.get('id') - or uuid.uuid4().hex) - tenant_ref['id'] = tenant_id - - tenant = self.identity_api.create_tenant( - context, tenant_id=tenant_id, data=tenant_ref) - return {'tenant': tenant} + def _format_authenticate(self, token_ref, roles_ref, catalog_ref): + o = self._format_token(token_ref, roles_ref) + o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) + return o def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] @@ -247,11 +298,6 @@ class KeystoneController(service.BaseApplication): o['access']['token']['tenant'] = token_ref['tenant'] return o - def _format_authenticate(self, token_ref, roles_ref, catalog_ref): - o = self._format_token(token_ref, roles_ref) - o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) - return o - def _format_catalog(self, catalog_ref): """KeystoneLight catalogs look like: @@ -296,6 +342,73 @@ class KeystoneController(service.BaseApplication): return services.values() + +class KeystoneTenantController(service.BaseApplication): + def __init__(self, options): + self.options = options + self.identity_api = identity.Manager(options) + self.policy_api = policy.Manager(options) + self.token_api = token.Manager(options) + + def get_tenants_for_token(self, context, **kw): + """Get valid tenants for token based on token used to authenticate. + + Pulls the token from the context, validates it and gets the valid + tenants for the user in the token. + + Doesn't care about token scopedness. + + """ + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + assert token_ref is not None + + user_ref = token_ref['user'] + tenant_refs = [] + for tenant_id in user_ref['tenants']: + tenant_refs.append(self.identity_api.get_tenant( + context=context, + tenant_id=tenant_id)) + return self._format_tenants_for_token(tenant_refs) + + def get_tenant(self, context, tenant_id): + # TODO(termie): this stuff should probably be moved to middleware + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['extras'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + + tenant = self.identity_api.get_tenant(context, tenant_id) + return {'tenant': tenant} + + def create_tenant(self, context, **kw): + # TODO(termie): this stuff should probably be moved to middleware + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['extras'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + tenant_ref = kw.get('tenant') + tenant_id = (tenant_ref.get('id') + and tenant_ref.get('id') + or uuid.uuid4().hex) + tenant_ref['id'] = tenant_id + + tenant = self.identity_api.create_tenant( + context, tenant_id=tenant_id, data=tenant_ref) + return {'tenant': tenant} + def _format_tenants_for_token(self, tenant_refs): for x in tenant_refs: x['enabled'] = True @@ -304,7 +417,52 @@ class KeystoneController(service.BaseApplication): return o -def app_factory(global_conf, **local_conf): +class KeystoneUserController(service.BaseApplication): + def __init__(self, options): + self.options = options + + def get_user(self, context, user_id): + raise NotImplemented() + + def get_version_info(self, context, module='version'): + # TODO(devcamcar): Pull appropriate module version and output. + raise NotImplemented() + + def get_extensions_info(self, context): + raise NotImplemented() + + +class KeystoneRoleController(service.BaseApplication): + def __init__(self, options): + self.options = options + + def get_user_roles(self, context, user_id, tenant_id=None): + raise NotImplemented() + + +class KeystoneVersionController(service.BaseApplication): + def __init__(self, options): + self.options = options + + def get_version_info(self, context, module='version'): + raise NotImplemented() + + +class KeystoneExtensionsController(service.BaseApplication): + def __init__(self, options): + self.options = options + + def get_extensions_info(self, context): + raise NotImplemented() + + +def service_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return KeystoneRouter(conf) + return KeystoneServiceRouter(conf) + + +def admin_app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return KeystoneAdminRouter(conf) diff --git a/keystonelight/test.py b/keystonelight/test.py index 282ad2d300..b2a5cc7f35 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -142,24 +142,25 @@ class TestCase(unittest.TestCase): extras['user'], extras['tenant'], models.Extras(**extras_ref)) setattr(self, 'extras_%s%s' % (extras['user'], extras['tenant']), rv) - def loadapp(self, config): + def loadapp(self, config, name='main'): if not config.startswith('config:'): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) - return deploy.loadapp(config) + return deploy.loadapp(config, name=name) def appconfig(self, config): if not config.startswith('config:'): config = 'config:%s.conf' % os.path.join(TESTSDIR, config) return deploy.appconfig(config) - def serveapp(self, config): - app = self.loadapp(config) - server = wsgi.Server() - server.start(app, 0, key='socket') + def serveapp(self, config, name=None): + app = self.loadapp(config, name=name) + server = wsgi.Server(app, 0) + server.start(key='socket') # Service catalog tests need to know the port we ran on. port = server.socket_info['socket'][1] self._update_server_options(server, 'public_port', port) + self._update_server_options(server, 'admin_port', port) return server def _update_server_options(self, server, key, value): diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index a3b4ccaa19..5ba1d5ce29 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -20,7 +20,6 @@ """Utility methods for working with WSGI servers.""" import logging -import os import sys import eventlet @@ -32,8 +31,6 @@ import webob import webob.dec import webob.exc -from paste import deploy - class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -49,17 +46,20 @@ class WritableLogger(object): class Server(object): """Server class to manage multiple WSGI sockets and applications.""" - def __init__(self, threads=1000): + def __init__(self, application, port, threads=1000): + self.application = application + self.port = port self.pool = eventlet.GreenPool(threads) self.socket_info = {} - def start(self, application, port, host='0.0.0.0', key=None, backlog=128): + def start(self, host='0.0.0.0', key=None, backlog=128): """Run a WSGI server with the given application.""" - self.application = application - arg0 = sys.argv[0] - logging.debug('Starting %(arg0)s on %(host)s:%(port)s' % locals()) - socket = eventlet.listen((host, port), backlog=backlog) - self.pool.spawn_n(self._run, application, socket) + logging.debug('Starting %(arg0)s on %(host)s:%(port)s' % \ + {'arg0': sys.argv[0], + 'host': host, + 'port': self.port}) + socket = eventlet.listen((host, self.port), backlog=backlog) + self.pool.spawn_n(self._run, self.application, socket) if key: self.socket_info[key] = socket.getsockname() diff --git a/tests/default.conf b/tests/default.conf index 0a3898f9c2..9cf2fb5d6a 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -4,7 +4,7 @@ identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 -admin_port = 5001 +admin_port = 35357 admin_token = ADMIN # config for TemplatedCatalog, using camelCase because I don't want to do @@ -37,16 +37,22 @@ paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[app:keystone] -paste.app_factory = keystonelight.keystone_compat:app_factory +[app:keystone_service] +paste.app_factory = keystonelight.keystone_compat:service_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +[app:keystone_admin] +paste.app_factory = keystonelight.keystone_compat:admin_app_factory [pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_service_api] +pipeline = token_auth admin_token_auth json_body debug keystone_service + +[pipeline:keystone_admin_api] +pipeline = token_auth admin_token_auth json_body debug keystone_admin + [composite:main] use = egg:Paste#urlmap / = keystonelight_api -/v2.0 = keystone_api +/v2.0 = keystone_service_api diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index ba893fbc03..f50c7706d6 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -4,6 +4,7 @@ identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 +admin_port = 35357 admin_token = ADMIN [filter:debug] @@ -21,16 +22,22 @@ paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[app:keystone] -paste.app_factory = keystonelight.keystone_compat:app_factory +[app:keystone_service] +paste.app_factory = keystonelight.keystone_compat:service_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +[app:keystone_admin] +paste.app_factory = keystonelight.keystone_compat:admin_app_factory [pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_service_api] +pipeline = token_auth admin_token_auth json_body debug keystone_service + +[pipeline:keystone_admin_api] +pipeline = token_auth admin_token_auth json_body debug keystone_admin + [composite:main] use = egg:Paste#urlmap / = keystonelight_api -/v2.0 = keystone_api +/v2.0 = keystone_service_api diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index 262215c2b2..c6e4ee06b5 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -4,12 +4,13 @@ identity_driver = keystonelight.backends.kvs.KvsIdentity token_driver = keystonelight.backends.kvs.KvsToken policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 +admin_port = 35357 admin_token = ADMIN # config for TemplatedCatalog, using camelCase because I don't want to do # translations for keystone compat catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 catalog.RegionOne.identity.name = 'Identity Service' @@ -36,16 +37,26 @@ paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory [app:keystonelight] paste.app_factory = keystonelight.service:app_factory -[app:keystone] -paste.app_factory = keystonelight.keystone_compat:app_factory +[app:keystone_service] +paste.app_factory = keystonelight.keystone_compat:service_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +[app:keystone_admin] +paste.app_factory = keystonelight.keystone_compat:admin_app_factory [pipeline:keystonelight_api] pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_service_api] +pipeline = token_auth admin_token_auth json_body debug keystone_service + +[pipeline:keystone_admin_api] +pipeline = token_auth admin_token_auth json_body debug keystone_admin + [composite:main] use = egg:Paste#urlmap / = keystonelight_api -/v2.0 = keystone_api +/v2.0 = keystone_service_api + +[composite:admin] +use = egg:Paste#urlmap +/v2.0 = keystone_admin_api diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index b0e55a958e..c9d0ae28ac 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -1,11 +1,8 @@ import json -import uuid from keystonelight import client from keystonelight import models from keystonelight import test -from keystonelight import utils -from keystonelight.backends import kvs import default_fixtures diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index e0eec5f457..73bb861e1b 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -1,6 +1,6 @@ -from keystonelight import models +import logging + from keystonelight import test -from keystonelight import utils import default_fixtures @@ -12,19 +12,25 @@ class CompatTestCase(test.TestCase): def setUp(self): super(CompatTestCase, self).setUp() - def _url(self): - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - return "http://localhost:%s/v2.0/" % port + def _public_url(self): + public_port = self.public_server.socket_info['socket'][1] + self.options['public_port'] = public_port + return "http://localhost:%s/v2.0" % public_port + + def _admin_url(self): + admin_port = self.admin_server.socket_info['socket'][1] + self.options['admin_port'] = admin_port + return "http://localhost:%s/v2.0" % admin_port def _client(self, **kwargs): from keystoneclient.v2_0 import client as ks_client - port = self.server.socket_info['socket'][1] - self.options['public_port'] = port - kc = ks_client.Client(**kwargs) + kc = ks_client.Client(endpoint=self._admin_url(), + auth_url=self._public_url(), + **kwargs) kc.authenticate() + # have to manually overwrite the management url after authentication + kc.management_url = self._admin_url() return kc @@ -37,12 +43,19 @@ class MasterCompatTestCase(CompatTestCase): from keystoneclient.v2_0 import client as ks_client reload(ks_client) - self.app = self.loadapp('keystoneclient_compat_master') self.options = self.appconfig('keystoneclient_compat_master') + self.public_app = self.loadapp('keystoneclient_compat_master', + name='main') + self.admin_app = self.loadapp('keystoneclient_compat_master', + name='admin') + self.load_backends() self.load_fixtures(default_fixtures) - self.server = self.serveapp('keystoneclient_compat_master') + self.public_server = self.serveapp('keystoneclient_compat_master', + name='main') + self.admin_server = self.serveapp('keystoneclient_compat_master', + name='admin') # TODO(termie): is_admin is being deprecated once the policy stuff # is all working @@ -63,16 +76,14 @@ class MasterCompatTestCase(CompatTestCase): # client.authenticate() def test_authenticate_tenant_name_and_tenants(self): - client = self._client(auth_url=self._url(), - username='FOO', + client = self._client(username='FOO', password='foo2', tenant_name='BAR') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_tenant_id_and_tenants(self): - client = self._client(auth_url=self._url(), - username='FOO', + client = self._client(username='FOO', password='foo2', tenant_id='bar') tenants = client.tenants.list() @@ -84,8 +95,7 @@ class MasterCompatTestCase(CompatTestCase): # FIXME(ja): add a test that admin endpoint returns unauthorized if not # admin def test_tenant_create(self): - client = self._client(auth_url=self._url(), - username='FOO', + client = self._client(username='FOO', password='foo2', tenant_name='BAR') client.tenants.create( From 3eb2adfff686a539c3f52afda2f45cd5aa4d1291 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Fri, 30 Dec 2011 16:14:36 -0800 Subject: [PATCH 109/334] Added broken tests to show compatibility gaps --- tests/test_keystoneclient_compat.py | 164 ++++++++++++++++++++++++---- 1 file changed, 141 insertions(+), 23 deletions(-) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 73bb861e1b..1cd10502cc 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -1,5 +1,3 @@ -import logging - from keystonelight import test import default_fixtures @@ -65,39 +63,159 @@ class MasterCompatTestCase(CompatTestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) - # def test_authenticate(self): - # from keystoneclient.v2_0 import client as ks_client - # - # port = self.server.socket_info['socket'][1] - # client = ks_client.Client(auth_url="http://localhost:%s/v2.0" % port, - # username='foo', - # password='foo', - # project_id='bar') - # client.authenticate() + def foo_client(self): + return self._client(username='FOO', + password='foo2', + tenant_name='BAR') - def test_authenticate_tenant_name_and_tenants(self): + def test_authenticate(self): client = self._client(username='FOO', password='foo2', - tenant_name='BAR') + tenant_id='bar') + authenticated = client.authenticate() + self.assertTrue(authenticated) + + def test_authenticate_tenant_name_and_tenants(self): + client = self.foo_client() tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_tenant_id_and_tenants(self): - client = self._client(username='FOO', - password='foo2', - tenant_id='bar') + client = self.foo_client() tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + def test_endpoints(self): + raise NotImplementedError() + #endpoints = client.tokens.endpoints(token) + # FIXME(ja): this test should require the "keystone:admin" roled # (probably the role set via --keystone_admin_role flag) # FIXME(ja): add a test that admin endpoint is only sent to admin user # FIXME(ja): add a test that admin endpoint returns unauthorized if not # admin - def test_tenant_create(self): - client = self._client(username='FOO', - password='foo2', - tenant_name='BAR') - client.tenants.create( - "hello", description="My new tenant!", enabled=True) - # FIXME(ja): assert tenant was created + def test_tenant_create_update_and_delete(self): + from keystoneclient import exceptions as client_exceptions + + test_tenant = 'new_tenant' + client = self.foo_client() + tenant = client.tenants.create(test_tenant, + description="My new tenant!", + enabled=True) + self.assertEquals(tenant.name, test_tenant) + + tenant = client.tenants.get(tenant.id) + self.assertEquals(tenant.name, test_tenant) + + # TODO(devcamcar): update gives 404. why? + tenant = client.tenants.update(tenant.id, + tenant_name='new_tenant2', + enabled=False, + description='new description') + self.assertEquals(tenant.name, 'new_tenant2') + self.assertFalse(tenant.enabled) + self.assertEquals(tenant.description, 'new description') + + # TODO(devcamcar): delete gives 404. why? + client.tenants.delete(test_tenant) + self.assertRaises(client_exceptions.NotFound, client.tenants.get, + tenant.id) + + def test_tenant_list(self): + client = self.foo_client() + tenants = client.tenants.list() + self.assertEquals(len(tenants), 1) + + def test_tenant_add_user(self): + raise NotImplementedError() + #client.roles.add_user_to_tenant(tenant_id, user_id, role_id) + + def test_tenant_remove_user(self): + raise NotImplementedError() + #client.roles.remove_user_from_tenant(tenant_id, user_id, role_id) + + def test_user_create_update_delete(self): + from keystoneclient import exceptions as client_exceptions + + test_user = 'new_user' + client = self.foo_client() + user = client.users.create(test_user, 'password', 'user1@test.com') + self.assertEquals(user.name, test_user) + + user = client.users.get(user.id) + self.assertEquals(user.name, test_user) + + user = client.users.update_email(user, 'user2@test.com') + self.assertEquals(user.email, 'user2@test.com') + + user = client.users.update_enabled(user, False) + self.assertFalse(user.enabled) + + # TODO(devcamcar): How to assert this succeeded? + user = client.users.update_password(user, 'password2') + + # TODO(devcamcar): How to assert this succeeded? + user = client.users.update_tenant(user, 'bar') + + client.users.delete(user.id) + self.assertRaises(client_exceptions.NotFound, client.users.get, + user.id) + + def test_user_list(self): + client = self.foo_client() + users = client.users.list() + self.assertTrue(len(users) > 0) + + def test_role_get(self): + client = self.foo_client() + role = client.roles.get('keystone_admin') + self.assertEquals(role.name, 'keystone_admin') + + def test_role_create_and_delete(self): + from keystoneclient import exceptions as client_exceptions + + test_role = 'new_role' + client = self.foo_client() + role = client.roles.create(test_role) + self.assertEquals(role.name, test_role) + + role = client.roles.get(test_role) + self.assertEquals(role.name, test_role) + + client.roles.delete(test_role) + + self.assertRaises(client_exceptions.NotFound, client.roles.get, + test_role) + + def test_role_list(self): + client = self.foo_client() + roles = client.roles.list() + # TODO(devcamcar): This assert should be more specific. + self.assertTrue(len(roles) > 0) + + def test_roles_get_by_user(self): + client = self.foo_client() + roles = client.roles.get_user_role_refs('FOO') + self.assertTrue(len(roles) > 0) + + def test_service_create_and_delete(self): + from keystoneclient import exceptions as client_exceptions + + test_service = 'new_service' + client = self.foo_client() + service = client.services.create(test_service, 'test', 'test') + self.assertEquals(service.name, test_service) + + service = client.services.get(service.id) + self.assertEquals(service.name, test_service) + + client.services.delete(service.id) + self.assertRaises(client_exceptions.NotFound, client.services.get, + service.id) + + def test_service_list(self): + client = self.foo_client() + services = client.services.list() + # TODO(devcamcar): This assert should be more specific. + self.assertTrue(len(services) > 0) + From f2a9c51af0d4c5f38663ac425b551e47d1c8f3dd Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 29 Dec 2011 17:40:32 -0800 Subject: [PATCH 110/334] Split keystone compat by admin and service endpoints --- keystonelight/keystone_compat.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 443680b37a..5a61741458 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -97,14 +97,13 @@ class KeystoneServiceRouter(wsgi.Router): mapper.connect('/ec2tokens', controller=auth_controller, action='authenticate_ec2', - conditions=dict(method=['POST'])) + conditions=dict(methods=['POST'])) - # Tenant Operations tenant_controller = KeystoneTenantController(self.options) mapper.connect('/tenants', controller=tenant_controller, action='get_tenants_for_token', - conditions=dict(method=['GET'])) + conditions=dict(methods=['GET'])) # Miscellaneous version_controller = KeystoneVersionController(self.options) @@ -350,7 +349,7 @@ class KeystoneTenantController(service.BaseApplication): self.policy_api = policy.Manager(options) self.token_api = token.Manager(options) - def get_tenants_for_token(self, context, **kw): + def get_tenants_for_token(self, context): """Get valid tenants for token based on token used to authenticate. Pulls the token from the context, validates it and gets the valid From 2fb294f6804cb7105dd985bf53fda55ac8817dae Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 29 Dec 2011 20:13:20 -0800 Subject: [PATCH 111/334] All tests but create_tenant pass --- keystonelight/keystone_compat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 5a61741458..a24c1752cc 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -99,6 +99,10 @@ class KeystoneServiceRouter(wsgi.Router): action='authenticate_ec2', conditions=dict(methods=['POST'])) +<<<<<<< HEAD +======= + # Tenant Operations +>>>>>>> All tests but create_tenant pass tenant_controller = KeystoneTenantController(self.options) mapper.connect('/tenants', controller=tenant_controller, @@ -349,7 +353,7 @@ class KeystoneTenantController(service.BaseApplication): self.policy_api = policy.Manager(options) self.token_api = token.Manager(options) - def get_tenants_for_token(self, context): + def get_tenants_for_token(self, context, **kw): """Get valid tenants for token based on token used to authenticate. Pulls the token from the context, validates it and gets the valid From c5b1b6fb05736ef5699111614ef72576eaeb55a8 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 29 Dec 2011 20:51:11 -0800 Subject: [PATCH 112/334] Made tests use both service and admin endpoints --- tests/test_keystoneclient_compat.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 1cd10502cc..68453008f5 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -13,12 +13,22 @@ class CompatTestCase(test.TestCase): def _public_url(self): public_port = self.public_server.socket_info['socket'][1] self.options['public_port'] = public_port +<<<<<<< HEAD return "http://localhost:%s/v2.0" % public_port +======= + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + return "http://localhost:%s/v2.0/" % public_port +>>>>>>> Made tests use both service and admin endpoints def _admin_url(self): admin_port = self.admin_server.socket_info['socket'][1] self.options['admin_port'] = admin_port +<<<<<<< HEAD return "http://localhost:%s/v2.0" % admin_port +======= + # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not + return "http://localhost:%s/v2.0/" % admin_port +>>>>>>> Made tests use both service and admin endpoints def _client(self, **kwargs): from keystoneclient.v2_0 import client as ks_client @@ -68,7 +78,7 @@ class MasterCompatTestCase(CompatTestCase): password='foo2', tenant_name='BAR') - def test_authenticate(self): + def test_authenticate_tenant_name_and_tenants(self): client = self._client(username='FOO', password='foo2', tenant_id='bar') @@ -218,4 +228,3 @@ class MasterCompatTestCase(CompatTestCase): services = client.services.list() # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(services) > 0) - From 30a11465e3e7339b15ff06dbaf1644c3c6cdadf0 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 3 Jan 2012 11:40:37 -0800 Subject: [PATCH 113/334] fixup --- keystonelight/keystone_compat.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index a24c1752cc..e19d92fd0a 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -99,10 +99,7 @@ class KeystoneServiceRouter(wsgi.Router): action='authenticate_ec2', conditions=dict(methods=['POST'])) -<<<<<<< HEAD -======= # Tenant Operations ->>>>>>> All tests but create_tenant pass tenant_controller = KeystoneTenantController(self.options) mapper.connect('/tenants', controller=tenant_controller, @@ -279,7 +276,6 @@ class KeystoneTokenController(service.BaseApplication): def _format_authenticate(self, token_ref, roles_ref, catalog_ref): o = self._format_token(token_ref, roles_ref) o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) - return o def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] From 0e7f06d71ed635a1fe9a7a0cde484f61a66e1db9 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 3 Jan 2012 11:41:43 -0800 Subject: [PATCH 114/334] merge fixes --- tests/test_keystoneclient_compat.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 68453008f5..6d2f24588e 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -13,22 +13,12 @@ class CompatTestCase(test.TestCase): def _public_url(self): public_port = self.public_server.socket_info['socket'][1] self.options['public_port'] = public_port -<<<<<<< HEAD return "http://localhost:%s/v2.0" % public_port -======= - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - return "http://localhost:%s/v2.0/" % public_port ->>>>>>> Made tests use both service and admin endpoints def _admin_url(self): admin_port = self.admin_server.socket_info['socket'][1] self.options['admin_port'] = admin_port -<<<<<<< HEAD return "http://localhost:%s/v2.0" % admin_port -======= - # NOTE(termie): novaclient wants a "/" at the end, keystoneclient does not - return "http://localhost:%s/v2.0/" % admin_port ->>>>>>> Made tests use both service and admin endpoints def _client(self, **kwargs): from keystoneclient.v2_0 import client as ks_client From 63c79344445bafb828c1cec3ddfe65e0e482c6f7 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 3 Jan 2012 12:51:28 -0800 Subject: [PATCH 115/334] get some tests working again --- keystonelight/keystone_compat.py | 5 ++++- keystonelight/test.py | 10 +++++----- tests/test_keystoneclient_compat.py | 7 ------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index e19d92fd0a..8e5a15d061 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -1,7 +1,9 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # this is the web service frontend that emulates keystone +import logging import uuid + import routes from keystonelight import catalog @@ -233,7 +235,7 @@ class KeystoneTokenController(service.BaseApplication): roles_ref = [] for role_id in extras_ref.get('roles', []): roles_ref.append(self.identity_api.get_role(context, role_id)) - + logging.debug('TOKEN_REF %s', token_ref) return self._format_authenticate(token_ref, roles_ref, catalog_ref) def authenticate_ec2(self, context): @@ -276,6 +278,7 @@ class KeystoneTokenController(service.BaseApplication): def _format_authenticate(self, token_ref, roles_ref, catalog_ref): o = self._format_token(token_ref, roles_ref) o['access']['serviceCatalog'] = self._format_catalog(catalog_ref) + return o def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] diff --git a/keystonelight/test.py b/keystonelight/test.py index b2a5cc7f35..0d42593aa3 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -177,11 +177,11 @@ class TestCase(unittest.TestCase): or hasattr(last, 'application') or hasattr(last, 'options')): - logging.debug('UPDATE %s: O %s A %s AS %s', - last.__class__, - getattr(last, 'options', None), - getattr(last, 'application', None), - getattr(last, 'applications', None)) + #logging.debug('UPDATE %s: O %s A %s AS %s', + # last.__class__, + # getattr(last, 'options', None), + # getattr(last, 'application', None), + # getattr(last, 'applications', None)) if hasattr(last, 'options'): last.options[key] = value diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 6d2f24588e..c829d267be 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -68,13 +68,6 @@ class MasterCompatTestCase(CompatTestCase): password='foo2', tenant_name='BAR') - def test_authenticate_tenant_name_and_tenants(self): - client = self._client(username='FOO', - password='foo2', - tenant_id='bar') - authenticated = client.authenticate() - self.assertTrue(authenticated) - def test_authenticate_tenant_name_and_tenants(self): client = self.foo_client() tenants = client.tenants.list() From 23c6f49f874b5cb2dc7e94aa3b47136ee042bd32 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 3 Jan 2012 16:00:59 -0800 Subject: [PATCH 116/334] example crud extension for create_tenant --- keystonelight/keystone_compat.py | 13 ++++++++ keystonelight/wsgi.py | 42 +++++++++++++++++++++++++ tests/keystoneclient_compat_master.conf | 5 ++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 8e5a15d061..6ff95bd146 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -125,6 +125,19 @@ class KeystoneServiceRouter(wsgi.Router): super(KeystoneServiceRouter, self).__init__(mapper) +class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): + def __init__(self, application, options): + self.options = options + mapper = routes.Mapper() + tenant_controller = KeystoneTenantController(self.options) + mapper.connect('/tenants', + controller=tenant_controller, + action='create_tenant', + conditions=dict(method=['POST'])) + super(KeystoneAdminCrudExtension, self).__init__( + application, options, mapper) + + class KeystoneTokenController(service.BaseApplication): def __init__(self, options): self.options = options diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 5ba1d5ce29..0744fd32a8 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -308,3 +308,45 @@ class Router(object): return webob.exc.HTTPNotFound() app = match['controller'] return app + + +class ExtensionRouter(Router): + """A router that allows extensions to supplement or overwrite routes. + + Expects to be subclassed. + """ + def __init__(self, application, options, mapper): + self.options = options + self.application = application + + mapper.connect('{path_info:.*}', controller=self.application) + super(ExtensionRouter, self).__init__(mapper) + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = nova.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import nova.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + conf = global_config.copy() + conf.update(local_config) + return cls(app, conf) + return _factory diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index c6e4ee06b5..1930c50cae 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -34,6 +34,9 @@ paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +[filter:crud_extension] +paste.filter_factory = keystonelight.keystone_compat:KeystoneAdminCrudExtension.factory + [app:keystonelight] paste.app_factory = keystonelight.service:app_factory @@ -50,7 +53,7 @@ pipeline = token_auth admin_token_auth json_body debug keystonelight pipeline = token_auth admin_token_auth json_body debug keystone_service [pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug keystone_admin +pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin [composite:main] use = egg:Paste#urlmap From e39665091379cf3a9ec6d97696c2f41f6343923d Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 10:45:18 -0800 Subject: [PATCH 117/334] copy over the os-ksadm extension --- keystonelight/keystone_compat.py | 132 ++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 9 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 6ff95bd146..c7fc5cdbe8 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -45,10 +45,6 @@ class KeystoneAdminRouter(wsgi.Router): controller=tenant_controller, action='get_tenant', conditions=dict(method=['GET'])) - mapper.connect('/tenants', - controller=tenant_controller, - action='create_tenant', - conditions=dict(method=['POST'])) # User Operations user_controller = KeystoneUserController(self.options) @@ -87,7 +83,6 @@ class KeystoneAdminRouter(wsgi.Router): class KeystoneServiceRouter(wsgi.Router): def __init__(self, options): self.options = options - mapper = routes.Mapper() # Token Operations @@ -126,14 +121,118 @@ class KeystoneServiceRouter(wsgi.Router): class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): + """Previously known as the OS-KSADM extension. + + Provides a bunch of CRUD operations for internal data types. + + """ + def __init__(self, application, options): self.options = options mapper = routes.Mapper() tenant_controller = KeystoneTenantController(self.options) - mapper.connect('/tenants', - controller=tenant_controller, - action='create_tenant', - conditions=dict(method=['POST'])) + user_controller = KeystoneUserController(self.options) + roles_controller = KeystoneRoleController(self.options) + services_controller = KeystoneServiceController(self.options) + + # Tenant Operations + mapper.connect("/tenants", controller=tenant_controller, + action="create_tenant", + conditions=dict(method=["POST"])) + mapper.connect("/tenants/{tenant_id}", + controller=tenant_controller, + action="update_tenant", conditions=dict(method=["POST"])) + mapper.connect("/tenants/{tenant_id}", + controller=tenant_controller, + action="delete_tenant", conditions=dict(method=["DELETE"])) + mapper.connect("/tenants/{tenant_id}/users", + controller=user_controller, + action="get_tenant_users", + conditions=dict(method=["GET"])) + + # User Operations + mapper.connect("/users", + controller=user_controller, + action="get_users", + conditions=dict(method=["GET"])) + mapper.connect("/users", + controller=user_controller, + action="create_user", + conditions=dict(method=["POST"])) + mapper.connect("/users/{user_id}", + controller=user_controller, + action="update_user", + conditions=dict(method=["POST"])) + mapper.connect("/users/{user_id}", + controller=user_controller, + action="delete_user", + conditions=dict(method=["DELETE"])) + + # NOTE(termie): not used and not necessary + #mapper.connect("/users/{user_id}/OS-KSADM/password", + # controller=user_controller, + # action="set_user_password", + # conditions=dict(method=["PUT"])) + #mapper.connect("/users/{user_id}/OS-KSADM/tenant", + # controller=user_controller, + # action="update_user_tenant", + # conditions=dict(method=["PUT"])) + # + # NOTE(termie): what does the next comment mean? + # Test this, test failed + #mapper.connect("/users/{user_id}/OS-KSADM/enabled", + # controller=user_controller, + # action="set_user_enabled", + # conditions=dict(method=["PUT"])) + + # User Roles + mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=roles_controller, action="add_role_to_user", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=roles_controller, action="delete_role_from_user", + conditions=dict(method=["DELETE"])) + + # User-Tenant Roles + mapper.connect( + "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=roles_controller, action="add_role_to_user", + conditions=dict(method=["PUT"])) + mapper.connect( + "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=roles_controller, action="delete_role_from_user", + conditions=dict(method=["DELETE"])) + + # Service Operations + mapper.connect("/OS-KSADM/services", + controller=services_controller, + action="get_services", + conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/services", + controller=services_controller, + action="create_service", + conditions=dict(method=["POST"])) + mapper.connect("/OS-KSADM/services/{service_id}", + controller=services_controller, + action="delete_service", + conditions=dict(method=["DELETE"])) + mapper.connect("/OS-KSADM/services/{service_id}", + controller=services_controller, + action="get_service", + conditions=dict(method=["GET"])) + + # Role Operations + mapper.connect("/OS-KSADM/roles", controller=roles_controller, + action="create_role", conditions=dict(method=["POST"])) + mapper.connect("/OS-KSADM/roles", controller=roles_controller, + action="get_roles", conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/roles/{role_id}", + controller=roles_controller, action="get_role", + conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/roles/{role_id}", + controller=roles_controller, action="delete_role", + conditions=dict(method=["DELETE"])) + super(KeystoneAdminCrudExtension, self).__init__( application, options, mapper) @@ -144,6 +243,7 @@ class KeystoneTokenController(service.BaseApplication): self.catalog_api = catalog.Manager(options) self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) + super(KeystoneTokenController, self).__init__() def authenticate(self, context, auth=None): """Authenticate credentials and return a token. @@ -364,6 +464,7 @@ class KeystoneTenantController(service.BaseApplication): self.identity_api = identity.Manager(options) self.policy_api = policy.Manager(options) self.token_api = token.Manager(options) + super(KeystoneTenantController, self).__init__() def get_tenants_for_token(self, context, **kw): """Get valid tenants for token based on token used to authenticate. @@ -435,6 +536,7 @@ class KeystoneTenantController(service.BaseApplication): class KeystoneUserController(service.BaseApplication): def __init__(self, options): self.options = options + super(KeystoneUserController, self).__init__() def get_user(self, context, user_id): raise NotImplemented() @@ -450,6 +552,16 @@ class KeystoneUserController(service.BaseApplication): class KeystoneRoleController(service.BaseApplication): def __init__(self, options): self.options = options + super(KeystoneRoleController, self).__init__() + + def get_user_roles(self, context, user_id, tenant_id=None): + raise NotImplemented() + + +class KeystoneServiceController(service.BaseApplication): + def __init__(self, options): + self.options = options + super(KeystoneServiceController, self).__init__() def get_user_roles(self, context, user_id, tenant_id=None): raise NotImplemented() @@ -458,6 +570,7 @@ class KeystoneRoleController(service.BaseApplication): class KeystoneVersionController(service.BaseApplication): def __init__(self, options): self.options = options + super(KeystoneVersionController, self).__init__() def get_version_info(self, context, module='version'): raise NotImplemented() @@ -466,6 +579,7 @@ class KeystoneVersionController(service.BaseApplication): class KeystoneExtensionsController(service.BaseApplication): def __init__(self, options): self.options = options + super(KeystoneExtensionsController, self).__init__() def get_extensions_info(self, context): raise NotImplemented() From 94e9d6bcc236f9538053f095e6f8d140c1f6d2e9 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 11:05:58 -0800 Subject: [PATCH 118/334] tenant test working again --- keystonelight/keystone_compat.py | 66 +++++++++++++++++------------ keystonelight/service.py | 14 ++++++ tests/test_keystoneclient_compat.py | 3 +- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index c7fc5cdbe8..2b1b9b45fa 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -5,6 +5,7 @@ import logging import uuid import routes +from webob import exc from keystonelight import catalog from keystonelight import identity @@ -132,8 +133,8 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): mapper = routes.Mapper() tenant_controller = KeystoneTenantController(self.options) user_controller = KeystoneUserController(self.options) - roles_controller = KeystoneRoleController(self.options) - services_controller = KeystoneServiceController(self.options) + role_controller = KeystoneRoleController(self.options) + service_controller = KeystoneServiceController(self.options) # Tenant Operations mapper.connect("/tenants", controller=tenant_controller, @@ -141,10 +142,12 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): conditions=dict(method=["POST"])) mapper.connect("/tenants/{tenant_id}", controller=tenant_controller, - action="update_tenant", conditions=dict(method=["POST"])) + action="update_tenant", + conditions=dict(method=["PUT"])) mapper.connect("/tenants/{tenant_id}", controller=tenant_controller, - action="delete_tenant", conditions=dict(method=["DELETE"])) + action="delete_tenant", + conditions=dict(method=["DELETE"])) mapper.connect("/tenants/{tenant_id}/users", controller=user_controller, action="get_tenant_users", @@ -162,7 +165,7 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): mapper.connect("/users/{user_id}", controller=user_controller, action="update_user", - conditions=dict(method=["POST"])) + conditions=dict(method=["PUT"])) mapper.connect("/users/{user_id}", controller=user_controller, action="delete_user", @@ -187,50 +190,50 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): # User Roles mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=roles_controller, action="add_role_to_user", + controller=role_controller, action="add_role_to_user", conditions=dict(method=["PUT"])) mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=roles_controller, action="delete_role_from_user", + controller=role_controller, action="delete_role_from_user", conditions=dict(method=["DELETE"])) # User-Tenant Roles mapper.connect( "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=roles_controller, action="add_role_to_user", + controller=role_controller, action="add_role_to_user", conditions=dict(method=["PUT"])) mapper.connect( "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=roles_controller, action="delete_role_from_user", + controller=role_controller, action="delete_role_from_user", conditions=dict(method=["DELETE"])) # Service Operations mapper.connect("/OS-KSADM/services", - controller=services_controller, + controller=service_controller, action="get_services", conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/services", - controller=services_controller, + controller=service_controller, action="create_service", conditions=dict(method=["POST"])) mapper.connect("/OS-KSADM/services/{service_id}", - controller=services_controller, + controller=service_controller, action="delete_service", conditions=dict(method=["DELETE"])) mapper.connect("/OS-KSADM/services/{service_id}", - controller=services_controller, + controller=service_controller, action="get_service", conditions=dict(method=["GET"])) # Role Operations - mapper.connect("/OS-KSADM/roles", controller=roles_controller, + mapper.connect("/OS-KSADM/roles", controller=role_controller, action="create_role", conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/roles", controller=roles_controller, + mapper.connect("/OS-KSADM/roles", controller=role_controller, action="get_roles", conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/roles/{role_id}", - controller=roles_controller, action="get_role", + controller=role_controller, action="get_role", conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/roles/{role_id}", - controller=roles_controller, action="delete_role", + controller=role_controller, action="delete_role", conditions=dict(method=["DELETE"])) super(KeystoneAdminCrudExtension, self).__init__( @@ -501,20 +504,13 @@ class KeystoneTenantController(service.BaseApplication): creds) tenant = self.identity_api.get_tenant(context, tenant_id) + if not tenant: + return exc.HTTPNotFound() return {'tenant': tenant} + # CRUD Extension def create_tenant(self, context, **kw): - # TODO(termie): this stuff should probably be moved to middleware - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['extras'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) + self.assert_admin(context) tenant_ref = kw.get('tenant') tenant_id = (tenant_ref.get('id') and tenant_ref.get('id') @@ -525,6 +521,20 @@ class KeystoneTenantController(service.BaseApplication): context, tenant_id=tenant_id, data=tenant_ref) return {'tenant': tenant} + def update_tenant(self, context, tenant_id, tenant): + self.assert_admin(context) + tenant_ref = self.identity_api.update_tenant( + context, tenant_id=tenant_id, data=tenant) + return {'tenant': tenant_ref} + + def delete_tenant(self, context, tenant_id, **kw): + self.assert_admin(context) + self.identity_api.delete_tenant(context, tenant_id=tenant_id) + + def get_tenant_users(self, context, **kw): + self.assert_admin(context) + pass + def _format_tenants_for_token(self, tenant_refs): for x in tenant_refs: x['enabled'] = True diff --git a/keystonelight/service.py b/keystonelight/service.py index c4b66f6d2d..ace3a1a191 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -72,9 +72,23 @@ class BaseApplication(wsgi.Application): if result is None or type(result) is str or type(result) is unicode: return result + elif isinstance(result, webob.exc.WSGIHTTPException): + return result return json.dumps(result) + def assert_admin(self, context): + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['extras'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index c829d267be..62a717be61 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -109,8 +109,7 @@ class MasterCompatTestCase(CompatTestCase): self.assertFalse(tenant.enabled) self.assertEquals(tenant.description, 'new description') - # TODO(devcamcar): delete gives 404. why? - client.tenants.delete(test_tenant) + client.tenants.delete(tenant.id) self.assertRaises(client_exceptions.NotFound, client.tenants.get, tenant.id) From c6d6d43b7e4af109423f23f6bbf136e94025437d Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 14:33:43 -0800 Subject: [PATCH 119/334] get tenant_add_and_remove_user test working this actually required a lot of stuff, basically we have to emulate some legacy features where user-tenant association is only handled through roles. in the process we added a few high level calls that will make abstracting the backend easier, too --- keystonelight/backends/kvs.py | 47 ++++++++++++++++ keystonelight/identity.py | 21 +++++++ keystonelight/keystone_compat.py | 85 ++++++++++++++++++++++++++++- tests/default_fixtures.py | 2 + tests/test_keystoneclient_compat.py | 31 +++++++++-- 5 files changed, 179 insertions(+), 7 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 37b1aa0282..439c4dee2d 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -63,6 +63,52 @@ class KvsIdentity(object): role_ref = self.db.get('role-%s' % role_id) return role_ref + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.add(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def remove_user_from_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.remove(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def get_tenants_for_user(self, user_id): + user_ref = self.get_user(user_id) + return user_ref.get('tenants', []) + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + return extras_ref.get('roles', []) + + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.add(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.remove(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + + + # CRUD def create_user(self, id, user): self.db.set('user-%s' % id, user) self.db.set('user_name-%s' % user['name'], user) @@ -101,6 +147,7 @@ class KvsIdentity(object): self.db.delete('tenant-%s' % id) return None + def create_extras(self, user_id, tenant_id, extras): self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) return extras diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 47d5403df6..dc76149478 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -37,6 +37,27 @@ class Manager(object): def get_role(self, context, role_id): return self.driver.get_role(role_id) + # These should probably be the high-level API calls + def add_user_to_tenant(self, context, user_id, tenant_id): + self.driver.add_user_to_tenant(user_id, tenant_id) + + def remove_user_from_tenant(self, context, user_id, tenant_id): + self.driver.remove_user_from_tenant(user_id, tenant_id) + + def get_tenants_for_user(self, context, user_id): + return self.driver.get_tenants_for_user(user_id) + + def get_roles_for_user_and_tenant(self, context, user_id, tenant_id): + return self.driver.get_roles_for_user_and_tenant(user_id, tenant_id) + + def add_role_to_user_and_tenant(self, context, user_id, tenant_id, role_id): + return self.driver.add_role_to_user_and_tenant(user_id, tenant_id, role_id) + + def remove_role_from_user_and_tenant(self, context, user_id, + tenant_id, role_id): + return self.driver.remove_role_from_user_and_tenant( + user_id, tenant_id, role_id) + # CRUD operations def create_user(self, context, user_id, data): return self.driver.create_user(user_id, data) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 2b1b9b45fa..1536bf8f75 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -2,6 +2,8 @@ # this is the web service frontend that emulates keystone import logging +import urllib +import urlparse import uuid import routes @@ -196,6 +198,17 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): controller=role_controller, action="delete_role_from_user", conditions=dict(method=["DELETE"])) + # COMPAT(diablo): User Roles + mapper.connect("/users/{user_id}/roleRefs", + controller=role_controller, action="get_role_refs", + conditions=dict(method=["GET"])) + mapper.connect("/users/{user_id}/roleRefs", + controller=role_controller, action="create_role_ref", + conditions=dict(method=["POST"])) + mapper.connect("/users/{user_id}/roleRefs/{role_ref_id}", + controller=role_controller, action="delete_role_ref", + conditions=dict(method=["DELETE"])) + # User-Tenant Roles mapper.connect( "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", @@ -533,7 +546,7 @@ class KeystoneTenantController(service.BaseApplication): def get_tenant_users(self, context, **kw): self.assert_admin(context) - pass + raise NotImplementedError() def _format_tenants_for_token(self, tenant_refs): for x in tenant_refs: @@ -562,11 +575,81 @@ class KeystoneUserController(service.BaseApplication): class KeystoneRoleController(service.BaseApplication): def __init__(self, options): self.options = options + self.identity_api = identity.Manager(options) + self.token_api = token.Manager(options) + self.policy_api = policy.Manager(options) super(KeystoneRoleController, self).__init__() def get_user_roles(self, context, user_id, tenant_id=None): raise NotImplemented() + # COMPAT(diablo): CRUD extension + def get_role_refs(self, context, user_id): + """Ultimate hack to get around having to make role_refs first-class. + + This will basically iterate over the various roles the user has in + all tenants the user is a member of and create fake role_refs where + the id encodes the user-tenant-role information so we can look + up the appropriate data when we need to delete them. + + """ + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + tenant_ids = self.identity_api.get_tenants_for_user(context, user_id) + o = [] + for tenant_id in tenant_ids: + role_ids = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + for role_id in role_ids: + ref = {'roleId': role_id, + 'tenantId': tenant_id, + 'userId': user_id} + ref['id'] = urllib.urlencode(ref) + o.append(ref) + return {'roles': o} + + def create_role_ref(self, context, user_id, role): + """This is actually used for adding a user to a tenant. + + In the legacy data model adding a user to a tenant required setting + a role. + + """ + self.assert_admin(context) + # TODO(termie): for now we're ignoring the actual role + tenant_id = role.get('tenantId') + role_id = role.get('roleId') + self.identity_api.add_user_to_tenant(context, tenant_id, user_id) + self.identity_api.add_role_to_user_and_tenant( + context, user_id, tenant_id, role_id) + role_ref = self.identity_api.get_role(context, role_id) + return {'role': role_ref} + + def delete_role_ref(self, context, user_id, role_ref_id): + """This is actually used for deleting a user from a tenant. + + In the legacy data model removing a user from a tenant required + deleting a role. + + To emulate this, we encode the tenant and role in the role_ref_id, + and if this happens to be the last role for the user-tenant pair, + we remove the user from the tenant. + + """ + self.assert_admin(context) + # TODO(termie): for now we're ignoring the actual role + role_ref_ref = urlparse.parse_qs(role_ref_id) + tenant_id = role_ref_ref.get('tenantId')[0] + role_id = role_ref_ref.get('roleId')[0] + self.identity_api.remove_role_from_user_and_tenant( + context, user_id, tenant_id, role_id) + roles = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + if not roles: + self.identity_api.remove_user_from_tenant( + context, tenant_id, user_id) + + class KeystoneServiceController(service.BaseApplication): def __init__(self, options): diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 8a6d6f3a77..7bddb607ca 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -1,5 +1,6 @@ TENANTS = [ {'id': 'bar', 'name': 'BAR'}, + {'id': 'baz', 'name': 'BAZ'}, ] USERS = [ @@ -12,4 +13,5 @@ EXTRAS = [ ROLES = [ {'id': 'keystone_admin', 'name': 'Keystone Admin'}, + {'id': 'useless', 'name': 'Useless'}, ] diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 62a717be61..3ebcad2e5b 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -118,13 +118,32 @@ class MasterCompatTestCase(CompatTestCase): tenants = client.tenants.list() self.assertEquals(len(tenants), 1) - def test_tenant_add_user(self): - raise NotImplementedError() - #client.roles.add_user_to_tenant(tenant_id, user_id, role_id) + def test_tenant_add_and_remove_user(self): + client = self.foo_client() + client.roles.add_user_to_tenant(self.tenant_baz['id'], + self.user_foo['id'], + self.role_useless['id']) + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] in + [x.id for x in tenant_refs]) - def test_tenant_remove_user(self): - raise NotImplementedError() - #client.roles.remove_user_from_tenant(tenant_id, user_id, role_id) + # get the "role_refs" so we get the proper id, this is how the clients + # do it + roleref_refs = client.roles.get_user_role_refs(self.user_foo['id']) + for roleref_ref in roleref_refs: + if (roleref_ref.roleId == self.role_useless['id'] and + roleref_ref.tenantId == self.tenant_baz['id']): + # use python's scope fall through to leave roleref_ref set + break + + + client.roles.remove_user_from_tenant(self.tenant_baz['id'], + self.user_foo['id'], + roleref_ref.id) + + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] not in + [x.id for x in tenant_refs]) def test_user_create_update_delete(self): from keystoneclient import exceptions as client_exceptions From ff15e5fca79611bd923da89fae873de58a23db9a Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 14:52:17 -0800 Subject: [PATCH 120/334] get endpoints test working --- keystonelight/keystone_compat.py | 6 +++--- tests/test_keystoneclient_compat.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 1536bf8f75..3ac1ce5d59 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -400,9 +400,9 @@ class KeystoneTokenController(service.BaseApplication): token_ref = self.token_api.get_token(context=context, token_id=token_id) catalog_ref = self.catalog_api.get_catalog(context, - token_ref['user_id'], - token_ref['tenant_id']) - return self._format_catalog(catalog_ref) + token_ref['user']['id'], + token_ref['tenant']['id']) + return {'token': {'serviceCatalog': self._format_catalog(catalog_ref)}} def _format_authenticate(self, token_ref, roles_ref, catalog_ref): o = self._format_token(token_ref, roles_ref) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 3ebcad2e5b..a992325a3e 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -78,9 +78,11 @@ class MasterCompatTestCase(CompatTestCase): tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + # TODO(termie): I'm not really sure that this is testing much def test_endpoints(self): - raise NotImplementedError() - #endpoints = client.tokens.endpoints(token) + client = self.foo_client() + token = client.auth_token + endpoints = client.tokens.endpoints(token) # FIXME(ja): this test should require the "keystone:admin" roled # (probably the role set via --keystone_admin_role flag) From 46943c5f4110042756f0919a45cff11544548c7b Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 15:47:52 -0800 Subject: [PATCH 121/334] get user tests working --- keystonelight/keystone_compat.py | 104 ++++++++++++++++++++++------ tests/test_keystoneclient_compat.py | 4 +- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 3ac1ce5d59..de2e3082e5 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -164,6 +164,7 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): controller=user_controller, action="create_user", conditions=dict(method=["POST"])) + # NOTE(termie): not in diablo mapper.connect("/users/{user_id}", controller=user_controller, action="update_user", @@ -173,22 +174,35 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): action="delete_user", conditions=dict(method=["DELETE"])) - # NOTE(termie): not used and not necessary - #mapper.connect("/users/{user_id}/OS-KSADM/password", - # controller=user_controller, - # action="set_user_password", - # conditions=dict(method=["PUT"])) - #mapper.connect("/users/{user_id}/OS-KSADM/tenant", - # controller=user_controller, - # action="update_user_tenant", - # conditions=dict(method=["PUT"])) - # - # NOTE(termie): what does the next comment mean? - # Test this, test failed - #mapper.connect("/users/{user_id}/OS-KSADM/enabled", - # controller=user_controller, - # action="set_user_enabled", - # conditions=dict(method=["PUT"])) + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/password", + controller=user_controller, + action="set_user_password", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/password", + controller=user_controller, + action="set_user_password", + conditions=dict(method=["PUT"])) + + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/tenant", + controller=user_controller, + action="update_user_tenant", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/tenant", + controller=user_controller, + action="update_user_tenant", + conditions=dict(method=["PUT"])) + + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/enabled", + controller=user_controller, + action="set_user_enabled", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/enabled", + controller=user_controller, + action="set_user_enabled", + conditions=dict(method=["PUT"])) # User Roles mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", @@ -259,6 +273,7 @@ class KeystoneTokenController(service.BaseApplication): self.catalog_api = catalog.Manager(options) self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) + self.policy_api = policy.Manager(options) super(KeystoneTokenController, self).__init__() def authenticate(self, context, auth=None): @@ -559,17 +574,61 @@ class KeystoneTenantController(service.BaseApplication): class KeystoneUserController(service.BaseApplication): def __init__(self, options): self.options = options + self.catalog_api = catalog.Manager(options) + self.identity_api = identity.Manager(options) + self.token_api = token.Manager(options) + self.policy_api = policy.Manager(options) super(KeystoneUserController, self).__init__() def get_user(self, context, user_id): - raise NotImplemented() + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + if not user_ref: + raise exc.HTTPNotFound() + return {'user': user_ref} - def get_version_info(self, context, module='version'): - # TODO(devcamcar): Pull appropriate module version and output. - raise NotImplemented() + # CRUD extension + def create_user(self, context, user): + self.assert_admin(context) + tenant_id = user.get('tenantId') + tenants = [] + if tenant_id: + tenants.append(tenant_id) + user_id = uuid.uuid4().hex + user_ref = user.copy() + #user_ref.pop('tenantId', None) + user_ref['id'] = user_id + user_ref['tenants'] = tenants + new_user_ref = self.identity_api.create_user( + context, user_id, user_ref) + return {'user': new_user_ref} + + # NOTE(termie): this is really more of a patch than a put + def update_user(self, context, user_id, user): + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + del user['id'] + user_ref.update(user) + self.identity_api.update_user(context, user_id, user_ref) + return {'user': user_ref} + + def delete_user(self, context, user_id): + self.assert_admin(context) + self.identity_api.delete_user(context, user_id) + + def set_user_enabled(self, context, user_id, user): + return self.update_user(context, user_id, user) + + def set_user_password(self, context, user_id, user): + return self.update_user(context, user_id, user) + + def update_user_tenant(self, context, user_id, user): + """Update the default tenant.""" + # ensure that we're a member of that tenant + tenant_id = user.get('tenantId') + self.identity_api.add_user_to_tenant(context, tenant_id, user_id) + return self.update_user(context, user_id, user) - def get_extensions_info(self, context): - raise NotImplemented() class KeystoneRoleController(service.BaseApplication): @@ -650,7 +709,6 @@ class KeystoneRoleController(service.BaseApplication): context, tenant_id, user_id) - class KeystoneServiceController(service.BaseApplication): def __init__(self, options): self.options = options diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index a992325a3e..b6f617d3f8 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -161,7 +161,9 @@ class MasterCompatTestCase(CompatTestCase): user = client.users.update_email(user, 'user2@test.com') self.assertEquals(user.email, 'user2@test.com') - user = client.users.update_enabled(user, False) + # NOTE(termie): update_enabled doesn't return anything, probably a bug + client.users.update_enabled(user, False) + user = client.users.get(user.id) self.assertFalse(user.enabled) # TODO(devcamcar): How to assert this succeeded? From 5c89972ffeb2256c362773cc367440edbe16f623 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 17:13:23 -0800 Subject: [PATCH 122/334] add list users --- keystonelight/backends/kvs.py | 9 +++++++++ keystonelight/identity.py | 5 +++++ keystonelight/keystone_compat.py | 8 +++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 439c4dee2d..9b98908656 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -63,6 +63,9 @@ class KvsIdentity(object): role_ref = self.db.get('role-%s' % role_id) return role_ref + def list_users(self): + return self.db.get('user_list', []) + # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): user_ref = self.get_user(user_id) @@ -112,6 +115,9 @@ class KvsIdentity(object): def create_user(self, id, user): self.db.set('user-%s' % id, user) self.db.set('user_name-%s' % user['name'], user) + user_list = set(self.db.get('user_list', [])) + user_list.add(id) + self.db.set('user_list', list(user_list)) return user def update_user(self, id, user): @@ -126,6 +132,9 @@ class KvsIdentity(object): old_user = self.db.get('user-%s' % id) self.db.delete('user_name-%s' % old_user['name']) self.db.delete('user-%s' % id) + user_list = set(self.db.get('user_list', [])) + user_list.remove(id) + self.db.set('user_list', list(user_list)) return None def create_tenant(self, id, tenant): diff --git a/keystonelight/identity.py b/keystonelight/identity.py index dc76149478..8e69a97b25 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -37,6 +37,11 @@ class Manager(object): def get_role(self, context, role_id): return self.driver.get_role(role_id) + # NOTE(termie): i think it will probably be a bad move in the end to try to + # list all users + def list_users(self, context): + return self.driver.list_users() + # These should probably be the high-level API calls def add_user_to_tenant(self, context, user_id, tenant_id): self.driver.add_user_to_tenant(user_id, tenant_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index de2e3082e5..1398af713a 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -587,6 +587,13 @@ class KeystoneUserController(service.BaseApplication): raise exc.HTTPNotFound() return {'user': user_ref} + def get_users(self, context): + # NOTE(termie): i can't imagine that this really wants all the data + # about every single user in the system... + self.assert_admin(context) + user_list = self.identity_api.list_users(context) + return {'users': [{'id': x} for x in user_list]} + # CRUD extension def create_user(self, context, user): self.assert_admin(context) @@ -630,7 +637,6 @@ class KeystoneUserController(service.BaseApplication): return self.update_user(context, user_id, user) - class KeystoneRoleController(service.BaseApplication): def __init__(self, options): self.options = options From ebe158f750cabe1786d2ab084746d32880ff98fa Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 17:31:54 -0800 Subject: [PATCH 123/334] add the various role tests --- keystonelight/backends/kvs.py | 9 ++++ keystonelight/identity.py | 3 ++ keystonelight/keystone_compat.py | 71 +++++++++++++++++++++-------- tests/test_keystoneclient_compat.py | 8 ++-- 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 9b98908656..89d9329821 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -66,6 +66,9 @@ class KvsIdentity(object): def list_users(self): return self.db.get('user_list', []) + def list_roles(self): + return self.db.get('role_list', []) + # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): user_ref = self.get_user(user_id) @@ -171,6 +174,9 @@ class KvsIdentity(object): def create_role(self, id, role): self.db.set('role-%s' % id, role) + role_list = set(self.db.get('role_list', [])) + role_list.add(id) + self.db.set('role_list', list(role_list)) return role def update_role(self, id, role): @@ -179,6 +185,9 @@ class KvsIdentity(object): def delete_role(self, id): self.db.delete('role-%s' % id) + role_list = set(self.db.get('role_list', [])) + role_list.remove(id) + self.db.set('role_list', list(role_list)) return None diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 8e69a97b25..52959ef979 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -42,6 +42,9 @@ class Manager(object): def list_users(self, context): return self.driver.list_users() + def list_roles(self, context): + return self.driver.list_roles() + # These should probably be the high-level API calls def add_user_to_tenant(self, context, user_id, tenant_id): self.driver.add_user_to_tenant(user_id, tenant_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 1398af713a..80a878de6c 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -235,33 +235,39 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): # Service Operations mapper.connect("/OS-KSADM/services", - controller=service_controller, - action="get_services", - conditions=dict(method=["GET"])) + controller=service_controller, + action="get_services", + conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/services", - controller=service_controller, - action="create_service", - conditions=dict(method=["POST"])) + controller=service_controller, + action="create_service", + conditions=dict(method=["POST"])) mapper.connect("/OS-KSADM/services/{service_id}", - controller=service_controller, - action="delete_service", - conditions=dict(method=["DELETE"])) + controller=service_controller, + action="delete_service", + conditions=dict(method=["DELETE"])) mapper.connect("/OS-KSADM/services/{service_id}", - controller=service_controller, - action="get_service", - conditions=dict(method=["GET"])) + controller=service_controller, + action="get_service", + conditions=dict(method=["GET"])) # Role Operations - mapper.connect("/OS-KSADM/roles", controller=role_controller, - action="create_role", conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/roles", controller=role_controller, - action="get_roles", conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/roles", + controller=role_controller, + action="create_role", + conditions=dict(method=["POST"])) + mapper.connect("/OS-KSADM/roles", + controller=role_controller, + action="get_roles", + conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/roles/{role_id}", - controller=role_controller, action="get_role", - conditions=dict(method=["GET"])) + controller=role_controller, + action="get_role", + conditions=dict(method=["GET"])) mapper.connect("/OS-KSADM/roles/{role_id}", - controller=role_controller, action="delete_role", - conditions=dict(method=["DELETE"])) + controller=role_controller, + action="delete_role", + conditions=dict(method=["DELETE"])) super(KeystoneAdminCrudExtension, self).__init__( application, options, mapper) @@ -648,6 +654,31 @@ class KeystoneRoleController(service.BaseApplication): def get_user_roles(self, context, user_id, tenant_id=None): raise NotImplemented() + # CRUD extension + def get_role(self, context, role_id): + self.assert_admin(context) + role_ref = self.identity_api.get_role(context, role_id) + if not role_ref: + raise exc.HTTPNotFound() + return {'role': role_ref} + + def create_role(self, context, role): + role_id = uuid.uuid4().hex + role['id'] = role_id + role_ref = self.identity_api.create_role(context, role_id, role) + return {'role': role_ref} + + def delete_role(self, context, role_id): + self.assert_admin(context) + role_ref = self.identity_api.delete_role(context, role_id) + + def get_roles(self, context): + self.assert_admin(context) + roles = self.identity_api.list_roles(context) + # TODO(termie): probably inefficient at some point + return {'roles': [self.identity_api.get_role(context, x) + for x in roles]} + # COMPAT(diablo): CRUD extension def get_role_refs(self, context, user_id): """Ultimate hack to get around having to make role_refs first-class. diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index b6f617d3f8..49cbc813b9 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -184,7 +184,7 @@ class MasterCompatTestCase(CompatTestCase): def test_role_get(self): client = self.foo_client() role = client.roles.get('keystone_admin') - self.assertEquals(role.name, 'keystone_admin') + self.assertEquals(role.id, 'keystone_admin') def test_role_create_and_delete(self): from keystoneclient import exceptions as client_exceptions @@ -194,10 +194,10 @@ class MasterCompatTestCase(CompatTestCase): role = client.roles.create(test_role) self.assertEquals(role.name, test_role) - role = client.roles.get(test_role) + role = client.roles.get(role) self.assertEquals(role.name, test_role) - client.roles.delete(test_role) + client.roles.delete(role) self.assertRaises(client_exceptions.NotFound, client.roles.get, test_role) @@ -210,7 +210,7 @@ class MasterCompatTestCase(CompatTestCase): def test_roles_get_by_user(self): client = self.foo_client() - roles = client.roles.get_user_role_refs('FOO') + roles = client.roles.get_user_role_refs('foo') self.assertTrue(len(roles) > 0) def test_service_create_and_delete(self): From 205a7b9aed46c62567da6edc67e8303ba42ce9c5 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 4 Jan 2012 18:23:07 -0800 Subject: [PATCH 124/334] finished up services stuff i don't think it makes any sense though --- keystonelight/backends/kvs.py | 27 ++++++++++++++++++++++--- keystonelight/backends/templated.py | 5 ++++- keystonelight/catalog.py | 12 +++++++++++ keystonelight/keystone_compat.py | 31 +++++++++++++++++++++++++++-- keystonelight/service.py | 6 +++++- tests/test_keystoneclient_compat.py | 2 ++ 6 files changed, 76 insertions(+), 7 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 89d9329821..5bb8eddd27 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -112,8 +112,6 @@ class KvsIdentity(object): extras_ref['roles'] = list(roles) self.update_extras(user_id, tenant_id, extras_ref) - - # CRUD def create_user(self, id, user): self.db.set('user-%s' % id, user) @@ -159,7 +157,6 @@ class KvsIdentity(object): self.db.delete('tenant-%s' % id) return None - def create_extras(self, user_id, tenant_id, extras): self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) return extras @@ -223,6 +220,30 @@ class KvsCatalog(object): def get_catalog(self, user_id, tenant_id, extras=None): return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) + def get_service(self, service_id): + return self.db.get('service-%s' % service_id) + + def list_services(self): + return self.db.get('service_list', []) + + def create_service(self, id, service): + self.db.set('service-%s' % id, service) + service_list = set(self.db.get('service_list', [])) + service_list.add(id) + self.db.set('service_list', list(service_list)) + return service + + def update_service(self, id, service): + self.db.set('service-%s' % id, service) + return service + + def delete_service(self, id): + self.db.delete('service-%s' % id) + service_list = set(self.db.get('service_list', [])) + service_list.remove(id) + self.db.set('service_list', list(service_list)) + return None + # Private interface def _create_catalog(self, user_id, tenant_id, data): self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index 528b8411ff..4ba5575851 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -1,7 +1,8 @@ from keystonelight import logging +from keystonelight.backends import kvs -class TemplatedCatalog(object): +class TemplatedCatalog(kvs.KvsCatalog): """A backend that generates endpoints for the Catalog based on templates. It is usually configured via config entries that look like: @@ -38,6 +39,8 @@ class TemplatedCatalog(object): else: self._load_templates(options) + super(TemplatedCatalog, self).__init__(options) + def _load_templates(self, options): o = {} for k, v in options.iteritems(): diff --git a/keystonelight/catalog.py b/keystonelight/catalog.py index f7df4f66d4..382b00ffaa 100644 --- a/keystonelight/catalog.py +++ b/keystonelight/catalog.py @@ -14,3 +14,15 @@ class Manager(object): def get_catalog(self, context, user_id, tenant_id, extras=None): """Return info for a catalog if it is valid.""" return self.driver.get_catalog(user_id, tenant_id, extras=extras) + + def get_service(self, context, service_id): + return self.driver.get_service(service_id) + + def list_services(self, context): + return self.driver.list_services() + + def create_service(self, context, service_id, data): + return self.driver.create_service(service_id, data) + + def delete_service(self, context, service_id): + return self.driver.delete_service(service_id) diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 80a878de6c..0fd998e196 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -646,6 +646,7 @@ class KeystoneUserController(service.BaseApplication): class KeystoneRoleController(service.BaseApplication): def __init__(self, options): self.options = options + self.catalog_api = catalog.Manager(options) self.identity_api = identity.Manager(options) self.token_api = token.Manager(options) self.policy_api = policy.Manager(options) @@ -749,10 +750,36 @@ class KeystoneRoleController(service.BaseApplication): class KeystoneServiceController(service.BaseApplication): def __init__(self, options): self.options = options + self.catalog_api = catalog.Manager(options) + self.identity_api = identity.Manager(options) + self.token_api = token.Manager(options) + self.policy_api = policy.Manager(options) super(KeystoneServiceController, self).__init__() - def get_user_roles(self, context, user_id, tenant_id=None): - raise NotImplemented() + # CRUD extensions + # NOTE(termie): this OS-KSADM stuff is about the lamest ever + def get_services(self, context): + service_list = self.catalog_api.list_services(context) + service_refs = [self.catalog_api.get_service(context, x) + for x in service_list] + return {'OS-KSADM:services': service_refs} + + def get_service(self, context, service_id): + service_ref = self.catalog_api.get_service(context, service_id) + if not service_ref: + raise exc.HTTPNotFound() + return {'OS-KSADM:service': service_ref} + + def delete_service(self, context, service_id): + service_ref = self.catalog_api.delete_service(context, service_id) + + def create_service(self, context, OS_KSADM_service): + service_id = uuid.uuid4().hex + service_ref = OS_KSADM_service.copy() + service_ref['id'] = service_id + new_service_ref = self.catalog_api.create_service( + context, service_id, service_ref) + return {'OS-KSADM:service': new_service_ref} class KeystoneVersionController(service.BaseApplication): diff --git a/keystonelight/service.py b/keystonelight/service.py index ace3a1a191..210250cb6d 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -67,7 +67,8 @@ class BaseApplication(wsgi.Application): method = getattr(self, action) # NOTE(vish): make sure we have no unicode keys for py2.6. - params = dict([(str(k), v) for (k, v) in params.iteritems()]) + params = dict([(self._normalize_arg(k), v) + for (k, v) in params.iteritems()]) result = method(context, **params) if result is None or type(result) is str or type(result) is unicode: @@ -77,6 +78,9 @@ class BaseApplication(wsgi.Application): return json.dumps(result) + def _normalize_arg(self, arg): + return str(arg).replace(':', '_').replace('-', '_') + def assert_admin(self, context): if not context['is_admin']: user_token_ref = self.token_api.get_token( diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 49cbc813b9..04714a53ab 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -230,6 +230,8 @@ class MasterCompatTestCase(CompatTestCase): def test_service_list(self): client = self.foo_client() + test_service = 'new_service' + service = client.services.create(test_service, 'test', 'test') services = client.services.list() # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(services) > 0) From 9691c0f7c8d1528290543765a355d0459f639e38 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Fri, 6 Jan 2012 15:58:02 -0800 Subject: [PATCH 125/334] tweaking for running regular tests in jenkins --- .gitignore | 4 ++++ run_tests.sh | 2 +- tools/pip-requires-test | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 540bc6e59a..bf562ab7f1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ vendor .ksl-venv run_tests.log +.coverage +covhtml +pep8.txt +nosetests.xml diff --git a/run_tests.sh b/run_tests.sh index 02c4458330..d561a7b925 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -90,7 +90,7 @@ function run_pep8 { # Just run PEP8 in current environment ${wrapper} pep8 --repeat --show-pep8 --show-source \ --ignore=E202,E111 \ - --exclude=vcsversion.py ${srcfiles} + --exclude=vcsversion.py ${srcfiles} | tee pep8.txt } NOSETESTS="python run_tests.py $noseopts $noseargs" diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 25b20295f9..94adcc0570 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -8,6 +8,7 @@ routes # keystonelight testing dependencies nose +nosexcover # for python-keystoneclient httplib2 From b766165b8b2ec4781c45feb53075bd338fb39b9f Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 28 Dec 2011 16:24:37 -0800 Subject: [PATCH 126/334] add sql backend, WIP --- keystonelight/backends/sql.py | 135 ++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 keystonelight/backends/sql.py diff --git a/keystonelight/backends/sql.py b/keystonelight/backends/sql.py new file mode 100644 index 0000000000..b568f3a50f --- /dev/null +++ b/keystonelight/backends/sql.py @@ -0,0 +1,135 @@ +import json + +import sqlalchemy as sql +from sqlalchemy import types as sql_types +from sqlalchemy.ext import declarative + +from keystonelight import models + + +Base = declarative_base() + + +class JsonBlob(sql_types.TypeDecorator): + impl = sql.Text + + def process_bind_param(self, value, dialect): + return json.dumps(value) + + def process_result_value(self, value, dialect): + return json.loads(value) + + +class DictBase(Base): + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __iter__(self): + self._i = iter(object_mapper(self).columns) + return self + + def next(self): + n = self._i.next().name + return n, getattr(self, n) + + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict. + + Includes attributes from joins. + + """ + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() + + +class User(Base): + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + + +class Tenant(Base): + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + + +class Role(Base): + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64)) + + +class Extras(Base): + __table_args__ = ( + sql.Index('idx_extras_usertenant', 'user', 'tenant'), + ) + + user = sql.Column(sql.String(64)) + tenant = sql.Column(sql.String(64)) + data = sql.Column(JsonBlob()) + + +class Token(Base): + id = sql.Column(sql.String(64), primary_key=True) + data = sql.Column(JsonBlob()) + + +class SqlIdentity(object): + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. + + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. + + """ + user_ref = self.get_user(user_id) + tenant_ref = None + extras_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') + if tenant_id and tenant_id not in user_ref['tenants']: + raise AssertionError('Invalid tenant') + + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + extras_ref = self.get_extras(user_id, tenant_id) + else: + extras_ref = {} + return (user_ref, tenant_ref, extras_ref) + + def get_tenant(self, tenant_id): + tenant_ref = self.query(Tenant).filter(Tenant.id == tenant_id) + return models.Tenant(**tenant_ref) + + def get_tenant_by_name(self, tenant_name): + tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + return tenant_ref + + def get_user(self, user_id): + user_ref = self.db.get('user-%s' % user_id) + return user_ref + + def get_user_by_name(self, user_name): + user_ref = self.db.get('user_name-%s' % user_name) + return user_ref + + def get_extras(self, user_id, tenant_id): + return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + + def get_role(self, role_id): + role_ref = self.db.get('role-%s' % role_id) + return role_ref + + From 775b8ed73415716f55becd7d0c94b48113f148d5 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 5 Jan 2012 14:21:18 -0800 Subject: [PATCH 127/334] get the sql ball rolling, still wip --- keystonelight/backends/sql.py | 246 +++++++++++++++++++++++++++++++--- keystonelight/models.py | 6 +- keystonelight/test.py | 6 +- tests/default.conf | 7 + tests/test_backend_kvs.py | 1 + 5 files changed, 244 insertions(+), 22 deletions(-) diff --git a/keystonelight/backends/sql.py b/keystonelight/backends/sql.py index b568f3a50f..38dc57e4c3 100644 --- a/keystonelight/backends/sql.py +++ b/keystonelight/backends/sql.py @@ -1,13 +1,19 @@ +"""SQL backends for the various services.""" + import json +import eventlet.db_pool import sqlalchemy as sql from sqlalchemy import types as sql_types from sqlalchemy.ext import declarative +import sqlalchemy.orm +import sqlalchemy.pool +import sqlalchemy.engine.url from keystonelight import models -Base = declarative_base() +Base = declarative.declarative_base() class JsonBlob(sql_types.TypeDecorator): @@ -20,7 +26,7 @@ class JsonBlob(sql_types.TypeDecorator): return json.loads(value) -class DictBase(Base): +class DictBase(object): def __setitem__(self, key, value): setattr(self, key, value) @@ -56,37 +62,105 @@ class DictBase(Base): return local.iteritems() -class User(Base): +class User(Base, DictBase): + __tablename__ = 'user' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + password = sql.Column(sql.String(64)) + + +class Tenant(Base, DictBase): + __tablename__ = 'tenant' id = sql.Column(sql.String(64), primary_key=True) name = sql.Column(sql.String(64), unique=True) -class Tenant(Base): - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - - -class Role(Base): +class Role(Base, DictBase): + __tablename__ = 'role' id = sql.Column(sql.String(64), primary_key=True) name = sql.Column(sql.String(64)) -class Extras(Base): +class Extras(Base, DictBase): + __tablename__ = 'extras' __table_args__ = ( sql.Index('idx_extras_usertenant', 'user', 'tenant'), ) + user = sql.Column(sql.String(64), primary_key=True) + tenant = sql.Column(sql.String(64), primary_key=True) + data = sql.Column(JsonBlob()) + + +class Token(Base, DictBase): + __tablename__ = 'token' + id = sql.Column(sql.String(64), primary_key=True) user = sql.Column(sql.String(64)) tenant = sql.Column(sql.String(64)) data = sql.Column(JsonBlob()) -class Token(Base): - id = sql.Column(sql.String(64), primary_key=True) - data = sql.Column(JsonBlob()) + +class SqlBase(object): + _MAKER = None + _ENGINE = None + + def __init__(self, options): + self.options = options + + def get_session(self, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy session.""" + if self._MAKER is None or self._ENGINE is None: + self._ENGINE = self.get_engine() + self._MAKER = self.get_maker(self._ENGINE, autocommit, expire_on_commit) + + session = self._MAKER() + # TODO(termie): we may want to do something similar + #session.query = nova.exception.wrap_db_error(session.query) + #session.flush = nova.exception.wrap_db_error(session.flush) + return session + + def get_engine(self): + """Return a SQLAlchemy engine.""" + connection_dict = sqlalchemy.engine.url.make_url( + self.options.get('sql_connection')) + + engine_args = { + "pool_recycle": self.options.get('sql_idle_timeout'), + "echo": False, + } + + if "sqlite" in connection_dict.drivername: + engine_args["poolclass"] = sqlalchemy.pool.NullPool + elif MySQLdb and "mysql" in connection_dict.drivername: + LOG.info(_("Using mysql/eventlet db_pool.")) + # MySQLdb won't accept 'None' in the password field + password = connection_dict.password or '' + pool_args = { + "db": connection_dict.database, + "passwd": password, + "host": connection_dict.host, + "user": connection_dict.username, + "min_size": self.options.get('sql_min_pool_size'), + "max_size": self.options.get('sql_max_pool_size'), + "max_idle": self.options.get('sql_idle_timeout'), + } + creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) + engine_args["pool_size"] = self.options.get('sql_max_pool_size') + engine_args["pool_timeout"] = self.options('sql_pool_timeout') + engine_args["creator"] = creator.create + + return sql.create_engine(self.options.get('sql_connection'), + **engine_args) + + def get_maker(self, engine, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy sessionmaker using the given engine.""" + return sqlalchemy.orm.sessionmaker(bind=engine, + autocommit=autocommit, + expire_on_commit=expire_on_commit) -class SqlIdentity(object): +class SqlIdentity(SqlBase): def authenticate(self, user_id=None, tenant_id=None, password=None): """Authenticate based on a user, tenant and password. @@ -110,7 +184,8 @@ class SqlIdentity(object): return (user_ref, tenant_ref, extras_ref) def get_tenant(self, tenant_id): - tenant_ref = self.query(Tenant).filter(Tenant.id == tenant_id) + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() return models.Tenant(**tenant_ref) def get_tenant_by_name(self, tenant_name): @@ -118,8 +193,9 @@ class SqlIdentity(object): return tenant_ref def get_user(self, user_id): - user_ref = self.db.get('user-%s' % user_id) - return user_ref + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + return models.User(**user_ref) def get_user_by_name(self, user_name): user_ref = self.db.get('user_name-%s' % user_name) @@ -132,4 +208,140 @@ class SqlIdentity(object): role_ref = self.db.get('role-%s' % role_id) return role_ref + def list_users(self): + return self.db.get('user_list', []) + def list_roles(self): + return self.db.get('role_list', []) + + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.add(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def remove_user_from_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.remove(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def get_tenants_for_user(self, user_id): + user_ref = self.get_user(user_id) + return user_ref.get('tenants', []) + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + return extras_ref.get('roles', []) + + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.add(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.remove(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + # CRUD + def create_user(self, id, user): + session = self.get_session() + session.add(User(**user)) + #self.db.set('user-%s' % id, user) + #self.db.set('user_name-%s' % user['name'], user) + #user_list = set(self.db.get('user_list', [])) + #user_list.add(id) + #self.db.set('user_list', list(user_list)) + return user + + def update_user(self, id, user): + # get the old name and delete it too + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.set('user-%s' % id, user) + self.db.set('user_name-%s' % user['name'], user) + return user + + def delete_user(self, id): + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.delete('user-%s' % id) + user_list = set(self.db.get('user_list', [])) + user_list.remove(id) + self.db.set('user_list', list(user_list)) + return None + + def create_tenant(self, id, tenant): + session = self.get_session() + session.add(Tenant(**tenant)) + #session.commit() + #self.db.set('tenant-%s' % id, tenant) + #self.db.set('tenant_name-%s' % tenant['name'], tenant) + return models.Tenant(**tenant) + + def update_tenant(self, id, tenant): + # get the old name and delete it too + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def delete_tenant(self, id): + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.delete('tenant-%s' % id) + return None + + def create_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + + def update_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + + def delete_extras(self, user_id, tenant_id): + self.db.delete('extras-%s-%s' % (tenant_id, user_id)) + return None + + def create_role(self, id, role): + self.db.set('role-%s' % id, role) + role_list = set(self.db.get('role_list', [])) + role_list.add(id) + self.db.set('role_list', list(role_list)) + return role + + def update_role(self, id, role): + self.db.set('role-%s' % id, role) + return role + + def delete_role(self, id): + self.db.delete('role-%s' % id) + role_list = set(self.db.get('role_list', [])) + role_list.remove(id) + self.db.set('role_list', list(role_list)) + return None + + + + +class SqlToken(SqlBase): + pass + +class SqlCatalog(SqlBase): + pass diff --git a/keystonelight/models.py b/keystonelight/models.py index 5cf76f6fea..c94af4c315 100644 --- a/keystonelight/models.py +++ b/keystonelight/models.py @@ -6,10 +6,8 @@ class Token(dict): class User(dict): - def __init__(self, id=None, tenants=None, *args, **kw): - if tenants is None: - tenants = [] - super(User, self).__init__(id=id, tenants=tenants, *args, **kw) + def __init__(self, id=None, *args, **kw): + super(User, self).__init__(id=id, *args, **kw) class Tenant(dict): diff --git a/keystonelight/test.py b/keystonelight/test.py index 0d42593aa3..0014aa5be9 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -125,7 +125,11 @@ class TestCase(unittest.TestCase): setattr(self, 'tenant_%s' % tenant['id'], rv) for user in fixtures.USERS: - rv = self.identity_api.create_user(user['id'], models.User(**user)) + user_copy = user.copy() + tenants = user_copy.pop('tenants') + rv = self.identity_api.create_user(user['id'], models.User(**user_copy)) + for tenant_id in tenants: + self.identity_api.add_user_to_tenant(tenant_id, user['id']) setattr(self, 'user_%s' % user['id'], rv) for role in fixtures.ROLES: diff --git a/tests/default.conf b/tests/default.conf index 9cf2fb5d6a..766b569ea4 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -21,6 +21,13 @@ catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v1.1/$(te catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s catalog.RegionOne.compute.name = 'Compute Service' +# for sql backends +sql_connection = sqlite:///:memory: +sql_idle_timeout = 200 +sql_min_pool_size = 5 +sql_max_pool_size = 10 +sql_pool_timeout = 200 + [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index d23605b61b..c88effdd0b 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -6,6 +6,7 @@ from keystonelight.backends import kvs import default_fixtures + class KvsIdentity(test.TestCase): def setUp(self): super(KvsIdentity, self).setUp() From 119808d60247ebe59cf84f89e76a262732da2bd9 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 5 Jan 2012 15:17:22 -0800 Subject: [PATCH 128/334] still wip, got migration mostly working --- keystonelight/backends/sql/__init__.py | 1 + .../backends/{sql.py => sql/core.py} | 10 +- .../backends/sql/migrate_repo/README | 4 + .../backends/sql/migrate_repo/__init__.py | 0 .../backends/sql/migrate_repo/manage.py | 5 + .../backends/sql/migrate_repo/migrate.cfg | 25 +++ .../versions/001_add_initial_tables.py | 14 ++ .../sql/migrate_repo/versions/__init__.py | 0 keystonelight/backends/sql/migration.py | 74 +++++++++ tests/default.conf | 2 +- tests/test_backend_kvs.py | 4 +- tests/test_backend_sql.py | 154 ++++++++++++++++++ 12 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 keystonelight/backends/sql/__init__.py rename keystonelight/backends/{sql.py => sql/core.py} (98%) create mode 100644 keystonelight/backends/sql/migrate_repo/README create mode 100644 keystonelight/backends/sql/migrate_repo/__init__.py create mode 100644 keystonelight/backends/sql/migrate_repo/manage.py create mode 100644 keystonelight/backends/sql/migrate_repo/migrate.cfg create mode 100644 keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py create mode 100644 keystonelight/backends/sql/migrate_repo/versions/__init__.py create mode 100644 keystonelight/backends/sql/migration.py create mode 100644 tests/test_backend_sql.py diff --git a/keystonelight/backends/sql/__init__.py b/keystonelight/backends/sql/__init__.py new file mode 100644 index 0000000000..38ecafd5d9 --- /dev/null +++ b/keystonelight/backends/sql/__init__.py @@ -0,0 +1 @@ +from keystonelight.backends.sql.core import * diff --git a/keystonelight/backends/sql.py b/keystonelight/backends/sql/core.py similarity index 98% rename from keystonelight/backends/sql.py rename to keystonelight/backends/sql/core.py index 38dc57e4c3..2fee455f80 100644 --- a/keystonelight/backends/sql.py +++ b/keystonelight/backends/sql/core.py @@ -27,6 +27,9 @@ class JsonBlob(sql_types.TypeDecorator): class DictBase(object): + def to_dict(self): + return dict(self.iteritems()) + def __setitem__(self, key, value): setattr(self, key, value) @@ -37,7 +40,7 @@ class DictBase(object): return getattr(self, key, default) def __iter__(self): - self._i = iter(object_mapper(self).columns) + self._i = iter(sqlalchemy.orm.object_mapper(self).columns) return self def next(self): @@ -186,7 +189,7 @@ class SqlIdentity(SqlBase): def get_tenant(self, tenant_id): session = self.get_session() tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - return models.Tenant(**tenant_ref) + return tenant_ref def get_tenant_by_name(self, tenant_name): tenant_ref = self.db.get('tenant_name-%s' % tenant_name) @@ -195,7 +198,7 @@ class SqlIdentity(SqlBase): def get_user(self, user_id): session = self.get_session() user_ref = session.query(User).filter_by(id=user_id).first() - return models.User(**user_ref) + return user_ref def get_user_by_name(self, user_name): user_ref = self.db.get('user_name-%s' % user_name) @@ -261,6 +264,7 @@ class SqlIdentity(SqlBase): def create_user(self, id, user): session = self.get_session() session.add(User(**user)) + session.flush() #self.db.set('user-%s' % id, user) #self.db.set('user_name-%s' % user['name'], user) #user_list = set(self.db.get('user_list', [])) diff --git a/keystonelight/backends/sql/migrate_repo/README b/keystonelight/backends/sql/migrate_repo/README new file mode 100644 index 0000000000..6218f8cac4 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/keystonelight/backends/sql/migrate_repo/__init__.py b/keystonelight/backends/sql/migrate_repo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystonelight/backends/sql/migrate_repo/manage.py b/keystonelight/backends/sql/migrate_repo/manage.py new file mode 100644 index 0000000000..39fa3892e5 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/keystonelight/backends/sql/migrate_repo/migrate.cfg b/keystonelight/backends/sql/migrate_repo/migrate.cfg new file mode 100644 index 0000000000..a8be608953 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=keystone + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py new file mode 100644 index 0000000000..e54106945a --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py @@ -0,0 +1,14 @@ +from sqlalchemy import * +from migrate import * + +from keystonelight.backends import sql + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + sql.Base.metadata.create_all(migrate_engine) + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + pass diff --git a/keystonelight/backends/sql/migrate_repo/versions/__init__.py b/keystonelight/backends/sql/migrate_repo/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystonelight/backends/sql/migration.py b/keystonelight/backends/sql/migration.py new file mode 100644 index 0000000000..1a4794b7e8 --- /dev/null +++ b/keystonelight/backends/sql/migration.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import sqlalchemy +from migrate.versioning import api as versioning_api + +try: + from migrate.versioning import exceptions as versioning_exceptions +except ImportError: + try: + # python-migration changed location of exceptions after 1.6.3 + # See LP Bug #717467 + from migrate import exceptions as versioning_exceptions + except ImportError: + sys.exit("python-migrate is not installed. Exiting.") + + +def db_sync(options, version=None): + if version is not None: + try: + version = int(version) + except ValueError: + raise Exception("version should be an integer") + + current_version = db_version(options) + repo_path = _find_migrate_repo() + if version is None or version > current_version: + return versioning_api.upgrade( + options.get('sql_connection'), repo_path, version) + else: + return versioning_api.downgrade( + options.get('sql_connection'), repo_path, version) + + +def db_version(options): + repo_path = _find_migrate_repo() + try: + return versioning_api.db_version( + options.get('sql_connection'), repo_path) + except versioning_exceptions.DatabaseNotControlledError: + return db_version_control(options, 0) + + +def db_version_control(options, version=None): + repo_path = _find_migrate_repo() + versioning_api.version_control( + options.get('sql_connection'), repo_path, version) + return version + + +def _find_migrate_repo(): + """Get the path for the migrate repository.""" + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'migrate_repo') + assert os.path.exists(path) + return path diff --git a/tests/default.conf b/tests/default.conf index 766b569ea4..295fe6ff53 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -22,7 +22,7 @@ catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$ catalog.RegionOne.compute.name = 'Compute Service' # for sql backends -sql_connection = sqlite:///:memory: +sql_connection = sqlite:///bla.db sql_idle_timeout = 200 sql_min_pool_size = 5 sql_max_pool_size = 10 diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index c88effdd0b..5c644d1007 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -11,8 +11,8 @@ class KvsIdentity(test.TestCase): def setUp(self): super(KvsIdentity, self).setUp() self.options = self.appconfig('default') - self.identity_api = kvs.KvsIdentity(options=self.options, db={}) - self.load_fixtures(default_fixtures) + #self.identity_api = kvs.KvsIdentity(options=self.options, db={}) + #self.load_fixtures(default_fixtures) def test_authenticate_bad_user(self): self.assertRaises(AssertionError, diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py new file mode 100644 index 0000000000..96f71eb67d --- /dev/null +++ b/tests/test_backend_sql.py @@ -0,0 +1,154 @@ +import os +import uuid + +from keystonelight import models +from keystonelight import test +from keystonelight.backends import sql +from keystonelight.backends.sql import migration + +import test_backend_kvs +import default_fixtures + + +class SqlIdentity(test_backend_kvs.KvsIdentity): + def setUp(self): + super(SqlIdentity, self).setUp() + self.options = self.appconfig('default') + os.unlink('bla.db') + migration.db_sync(self.options, 1) + self.identity_api = sql.SqlIdentity(options=self.options) + self.load_fixtures(default_fixtures) + + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') + + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) + + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(not extras_ref) + + def test_authenticate(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) + + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + self.assertDictEquals(user_ref, self.user_foo) + + def test_get_extras_bad_user(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(extras_ref is None) + + def test_get_extras_bad_tenant(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(extras_ref is None) + + def test_get_extras(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_role(self): + role_ref = self.identity_api.get_role( + role_id=self.role_keystone_admin['id']) + self.assertDictEquals(role_ref, self.role_keystone_admin) + + +class SqlToken(test_backend_kvs.KvsToken): + def setUp(self): + super(SqlToken, self).setUp() + self.token_api = sql.SqlToken(options=options) + self.load_fixtures(default_fixtures) + + def test_token_crud(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, + 'a': 'b'} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + + new_data_ref = self.token_api.get_token(token_id) + self.assertEquals(new_data_ref, data) + + self.token_api.delete_token(token_id) + deleted_data_ref = self.token_api.get_token(token_id) + self.assert_(deleted_data_ref is None) + + +class SqlCatalog(test_backend_kvs.KvsCatalog): + def setUp(self): + super(SqlCatalog, self).setUp() + self.catalog_api = sql.SqlCatalog(options=options) + self._load_fixtures() + + def _load_fixtures(self): + self.catalog_foobar = self.catalog_api._create_catalog( + 'foo', 'bar', + {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) + + def test_get_catalog_bad_user(self): + catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') + self.assert_(catalog_ref is None) + + def test_get_catalog_bad_tenant(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') + self.assert_(catalog_ref is None) + + def test_get_catalog(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar') + self.assertDictEquals(catalog_ref, self.catalog_foobar) From 6495d41ac2bf3548c4d3a9bea40d7ecd3ee8ac1c Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 17:00:01 -0800 Subject: [PATCH 129/334] most tests working again --- keystonelight/backends/sql/core.py | 132 +++++++++++++---------- keystonelight/models.py | 7 +- keystonelight/test.py | 9 +- tests/default_fixtures.py | 2 +- tests/test_backend_kvs.py | 96 +---------------- tests/test_backend_sql.py | 161 +++++++---------------------- 6 files changed, 126 insertions(+), 281 deletions(-) diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py index 2fee455f80..6bd9bcecec 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystonelight/backends/sql/core.py @@ -15,7 +15,7 @@ from keystonelight import models Base = declarative.declarative_base() - +# Special Fields class JsonBlob(sql_types.TypeDecorator): impl = sql.Text @@ -45,7 +45,7 @@ class DictBase(object): def next(self): n = self._i.next().name - return n, getattr(self, n) + return n def update(self, values): """Make the model object behave like a dict.""" @@ -64,7 +64,7 @@ class DictBase(object): local.update(joined) return local.iteritems() - +# Tables class User(Base, DictBase): __tablename__ = 'user' id = sql.Column(sql.String(64), primary_key=True) @@ -86,12 +86,12 @@ class Role(Base, DictBase): class Extras(Base, DictBase): __tablename__ = 'extras' - __table_args__ = ( - sql.Index('idx_extras_usertenant', 'user', 'tenant'), - ) + #__table_args__ = ( + # sql.Index('idx_extras_usertenant', 'user', 'tenant'), + # ) - user = sql.Column(sql.String(64), primary_key=True) - tenant = sql.Column(sql.String(64), primary_key=True) + user_id = sql.Column(sql.String(64), primary_key=True) + tenant_id = sql.Column(sql.String(64), primary_key=True) data = sql.Column(JsonBlob()) @@ -103,6 +103,17 @@ class Token(Base, DictBase): data = sql.Column(JsonBlob()) +class UserTenantMembership(Base, DictBase): + """Tenant membership join table.""" + __tablename__ = 'user_tenant_membership' + user_id = sql.Column(sql.String(64), + sql.ForeignKey('user.id'), + primary_key=True) + tenant_id = sql.Column(sql.String(64), + sql.ForeignKey('tenant.id'), + primary_key=True) + + class SqlBase(object): _MAKER = None @@ -135,23 +146,23 @@ class SqlBase(object): if "sqlite" in connection_dict.drivername: engine_args["poolclass"] = sqlalchemy.pool.NullPool - elif MySQLdb and "mysql" in connection_dict.drivername: - LOG.info(_("Using mysql/eventlet db_pool.")) - # MySQLdb won't accept 'None' in the password field - password = connection_dict.password or '' - pool_args = { - "db": connection_dict.database, - "passwd": password, - "host": connection_dict.host, - "user": connection_dict.username, - "min_size": self.options.get('sql_min_pool_size'), - "max_size": self.options.get('sql_max_pool_size'), - "max_idle": self.options.get('sql_idle_timeout'), - } - creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) - engine_args["pool_size"] = self.options.get('sql_max_pool_size') - engine_args["pool_timeout"] = self.options('sql_pool_timeout') - engine_args["creator"] = creator.create + #elif MySQLdb and "mysql" in connection_dict.drivername: + # LOG.info(_("Using mysql/eventlet db_pool.")) + # # MySQLdb won't accept 'None' in the password field + # password = connection_dict.password or '' + # pool_args = { + # "db": connection_dict.database, + # "passwd": password, + # "host": connection_dict.host, + # "user": connection_dict.username, + # "min_size": self.options.get('sql_min_pool_size'), + # "max_size": self.options.get('sql_max_pool_size'), + # "max_idle": self.options.get('sql_idle_timeout'), + # } + # creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) + # engine_args["pool_size"] = self.options.get('sql_max_pool_size') + # engine_args["pool_timeout"] = self.options('sql_pool_timeout') + # engine_args["creator"] = creator.create return sql.create_engine(self.options.get('sql_connection'), **engine_args) @@ -192,7 +203,8 @@ class SqlIdentity(SqlBase): return tenant_ref def get_tenant_by_name(self, tenant_name): - tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() return tenant_ref def get_user(self, user_id): @@ -201,29 +213,38 @@ class SqlIdentity(SqlBase): return user_ref def get_user_by_name(self, user_name): - user_ref = self.db.get('user_name-%s' % user_name) + session = self.get_session() + user_ref = session.query(User).filter_by(name=user_name).first() return user_ref def get_extras(self, user_id, tenant_id): - return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + session = self.get_session() + extras_ref = session.query(Extras)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + return getattr(extras_ref, 'data', None) def get_role(self, role_id): - role_ref = self.db.get('role-%s' % role_id) + session = self.get_session() + role_ref = session.query(Role).filter_by(id=role_id).first() return role_ref def list_users(self): - return self.db.get('user_list', []) + session = self.get_session() + user_refs = session.query(User) + return list(user_refs) def list_roles(self): - return self.db.get('role_list', []) + session = self.get_session() + role_refs = session.query(Role) + return list(role_refs) # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.add(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + session = self.get_session() + with session.begin(): + session.add(UserTenantMembership(user_id=user_id, tenant_id=tenant_id)) def remove_user_from_tenant(self, tenant_id, user_id): user_ref = self.get_user(user_id) @@ -233,8 +254,12 @@ class SqlIdentity(SqlBase): self.update_user(user_id, user_ref) def get_tenants_for_user(self, user_id): - user_ref = self.get_user(user_id) - return user_ref.get('tenants', []) + session = self.get_session() + membership_refs = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .all() + + return [x.tenant_id for x in membership_refs] def get_roles_for_user_and_tenant(self, user_id, tenant_id): extras_ref = self.get_extras(user_id, tenant_id) @@ -263,13 +288,8 @@ class SqlIdentity(SqlBase): # CRUD def create_user(self, id, user): session = self.get_session() - session.add(User(**user)) - session.flush() - #self.db.set('user-%s' % id, user) - #self.db.set('user_name-%s' % user['name'], user) - #user_list = set(self.db.get('user_list', [])) - #user_list.add(id) - #self.db.set('user_list', list(user_list)) + with session.begin(): + session.add(User(**user)) return user def update_user(self, id, user): @@ -291,11 +311,9 @@ class SqlIdentity(SqlBase): def create_tenant(self, id, tenant): session = self.get_session() - session.add(Tenant(**tenant)) - #session.commit() - #self.db.set('tenant-%s' % id, tenant) - #self.db.set('tenant_name-%s' % tenant['name'], tenant) - return models.Tenant(**tenant) + with session.begin(): + session.add(Tenant(**tenant)) + return tenant def update_tenant(self, id, tenant): # get the old name and delete it too @@ -312,7 +330,9 @@ class SqlIdentity(SqlBase): return None def create_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + session = self.get_session() + with session.begin(): + session.add(Extras(user_id=user_id, tenant_id=tenant_id, data=extras)) return extras def update_extras(self, user_id, tenant_id, extras): @@ -324,10 +344,9 @@ class SqlIdentity(SqlBase): return None def create_role(self, id, role): - self.db.set('role-%s' % id, role) - role_list = set(self.db.get('role_list', [])) - role_list.add(id) - self.db.set('role_list', list(role_list)) + session = self.get_session() + with session.begin(): + session.add(Role(**role)) return role def update_role(self, id, role): @@ -342,10 +361,9 @@ class SqlIdentity(SqlBase): return None - - class SqlToken(SqlBase): pass + class SqlCatalog(SqlBase): pass diff --git a/keystonelight/models.py b/keystonelight/models.py index c94af4c315..7cfec85c11 100644 --- a/keystonelight/models.py +++ b/keystonelight/models.py @@ -21,5 +21,8 @@ class Role(dict): class Extras(dict): - def __init__(self, user=None, tenant=None, *args, **kw): - super(Extras, self).__init__(user=user, tenant=tenant, *args, **kw) + def __init__(self, user_id=None, tenant_id=None, *args, **kw): + super(Extras, self).__init__(user_id=user_id, + tenant_id=tenant_id, + *args, + **kw) diff --git a/keystonelight/test.py b/keystonelight/test.py index 0014aa5be9..2045520a02 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -140,11 +140,12 @@ class TestCase(unittest.TestCase): extras_ref = extras.copy() # TODO(termie): these will probably end up in the model anyway, so this # may be futile - del extras_ref['user'] - del extras_ref['tenant'] + del extras_ref['user_id'] + del extras_ref['tenant_id'] rv = self.identity_api.create_extras( - extras['user'], extras['tenant'], models.Extras(**extras_ref)) - setattr(self, 'extras_%s%s' % (extras['user'], extras['tenant']), rv) + extras['user_id'], extras['tenant_id'], models.Extras(**extras_ref)) + setattr(self, + 'extras_%s%s' % (extras['user_id'], extras['tenant_id']), rv) def loadapp(self, config, name='main'): if not config.startswith('config:'): diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 7bddb607ca..0e24d750fc 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -8,7 +8,7 @@ USERS = [ ] EXTRAS = [ - {'user': 'foo', 'tenant': 'bar', 'extra': 'extra'}, + {'user_id': 'foo', 'tenant_id': 'bar', 'extra': 'extra'}, ] ROLES = [ diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 5c644d1007..089d088563 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -4,104 +4,16 @@ from keystonelight import models from keystonelight import test from keystonelight.backends import kvs +import test_backends import default_fixtures -class KvsIdentity(test.TestCase): +class KvsIdentity(test.TestCase, test_backends.IdentityTests): def setUp(self): super(KvsIdentity, self).setUp() self.options = self.appconfig('default') - #self.identity_api = kvs.KvsIdentity(options=self.options, db={}) - #self.load_fixtures(default_fixtures) - - def test_authenticate_bad_user(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - - def test_authenticate_bad_password(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password'] + 'WRONG') - - def test_authenticate_invalid_tenant(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG', - password=self.user_foo['password']) - - def test_authenticate_no_tenant(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assert_(tenant_ref is None) - self.assert_(not extras_ref) - - def test_authenticate(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assertDictEquals(tenant_ref, self.tenant_bar) - self.assertDictEquals(extras_ref, self.extras_foobar) - - def test_get_tenant_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(tenant_ref is None) - - def test_get_tenant(self): - tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_tenant_by_name_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['name'] + 'WRONG') - self.assert_(tenant_ref is None) - - def test_get_tenant_by_name(self): - tenant_ref = self.identity_api.get_tenant_by_name( - tenant_name=self.tenant_bar['name']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_user_bad_user(self): - user_ref = self.identity_api.get_user( - user_id=self.user_foo['id'] + 'WRONG') - self.assert_(user_ref is None) - - def test_get_user(self): - user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) - self.assertDictEquals(user_ref, self.user_foo) - - def test_get_extras_bad_user(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id']) - self.assert_(extras_ref is None) - - def test_get_extras_bad_tenant(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(extras_ref is None) - - def test_get_extras(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id']) - self.assertDictEquals(extras_ref, self.extras_foobar) - - def test_get_role(self): - role_ref = self.identity_api.get_role( - role_id=self.role_keystone_admin['id']) - self.assertDictEquals(role_ref, self.role_keystone_admin) + self.identity_api = kvs.KvsIdentity(options=self.options, db={}) + self.load_fixtures(default_fixtures) class KvsToken(test.TestCase): diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index 96f71eb67d..1f64d898f5 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -6,11 +6,11 @@ from keystonelight import test from keystonelight.backends import sql from keystonelight.backends.sql import migration -import test_backend_kvs +import test_backends import default_fixtures -class SqlIdentity(test_backend_kvs.KvsIdentity): +class SqlIdentity(test.TestCase, test_backends.IdentityTests): def setUp(self): super(SqlIdentity, self).setUp() self.options = self.appconfig('default') @@ -19,136 +19,47 @@ class SqlIdentity(test_backend_kvs.KvsIdentity): self.identity_api = sql.SqlIdentity(options=self.options) self.load_fixtures(default_fixtures) - def test_authenticate_bad_user(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - def test_authenticate_bad_password(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password'] + 'WRONG') +#class SqlToken(test_backend_kvs.KvsToken): +# def setUp(self): +# super(SqlToken, self).setUp() +# self.token_api = sql.SqlToken(options=options) +# self.load_fixtures(default_fixtures) - def test_authenticate_invalid_tenant(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG', - password=self.user_foo['password']) +# def test_token_crud(self): +# token_id = uuid.uuid4().hex +# data = {'id': token_id, +# 'a': 'b'} +# data_ref = self.token_api.create_token(token_id, data) +# self.assertDictEquals(data_ref, data) - def test_authenticate_no_tenant(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assert_(tenant_ref is None) - self.assert_(not extras_ref) +# new_data_ref = self.token_api.get_token(token_id) +# self.assertEquals(new_data_ref, data) - def test_authenticate(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - self.assertDictEquals(user_ref, self.user_foo) - self.assertDictEquals(tenant_ref, self.tenant_bar) - self.assertDictEquals(extras_ref, self.extras_foobar) - - def test_get_tenant_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(tenant_ref is None) - - def test_get_tenant(self): - tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_tenant_by_name_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['name'] + 'WRONG') - self.assert_(tenant_ref is None) - - def test_get_tenant_by_name(self): - tenant_ref = self.identity_api.get_tenant_by_name( - tenant_name=self.tenant_bar['name']) - self.assertDictEquals(tenant_ref, self.tenant_bar) - - def test_get_user_bad_user(self): - user_ref = self.identity_api.get_user( - user_id=self.user_foo['id'] + 'WRONG') - self.assert_(user_ref is None) - - def test_get_user(self): - user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) - self.assertDictEquals(user_ref, self.user_foo) - - def test_get_extras_bad_user(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id']) - self.assert_(extras_ref is None) - - def test_get_extras_bad_tenant(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(extras_ref is None) - - def test_get_extras(self): - extras_ref = self.identity_api.get_extras( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id']) - self.assertDictEquals(extras_ref, self.extras_foobar) - - def test_get_role(self): - role_ref = self.identity_api.get_role( - role_id=self.role_keystone_admin['id']) - self.assertDictEquals(role_ref, self.role_keystone_admin) +# self.token_api.delete_token(token_id) +# deleted_data_ref = self.token_api.get_token(token_id) +# self.assert_(deleted_data_ref is None) -class SqlToken(test_backend_kvs.KvsToken): - def setUp(self): - super(SqlToken, self).setUp() - self.token_api = sql.SqlToken(options=options) - self.load_fixtures(default_fixtures) +#class SqlCatalog(test_backend_kvs.KvsCatalog): +# def setUp(self): +# super(SqlCatalog, self).setUp() +# self.catalog_api = sql.SqlCatalog(options=options) +# self._load_fixtures() - def test_token_crud(self): - token_id = uuid.uuid4().hex - data = {'id': token_id, - 'a': 'b'} - data_ref = self.token_api.create_token(token_id, data) - self.assertDictEquals(data_ref, data) +# def _load_fixtures(self): +# self.catalog_foobar = self.catalog_api._create_catalog( +# 'foo', 'bar', +# {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) - new_data_ref = self.token_api.get_token(token_id) - self.assertEquals(new_data_ref, data) +# def test_get_catalog_bad_user(self): +# catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') +# self.assert_(catalog_ref is None) - self.token_api.delete_token(token_id) - deleted_data_ref = self.token_api.get_token(token_id) - self.assert_(deleted_data_ref is None) +# def test_get_catalog_bad_tenant(self): +# catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') +# self.assert_(catalog_ref is None) - -class SqlCatalog(test_backend_kvs.KvsCatalog): - def setUp(self): - super(SqlCatalog, self).setUp() - self.catalog_api = sql.SqlCatalog(options=options) - self._load_fixtures() - - def _load_fixtures(self): - self.catalog_foobar = self.catalog_api._create_catalog( - 'foo', 'bar', - {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) - - def test_get_catalog_bad_user(self): - catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') - self.assert_(catalog_ref is None) - - def test_get_catalog_bad_tenant(self): - catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') - self.assert_(catalog_ref is None) - - def test_get_catalog(self): - catalog_ref = self.catalog_api.get_catalog('foo', 'bar') - self.assertDictEquals(catalog_ref, self.catalog_foobar) +# def test_get_catalog(self): +# catalog_ref = self.catalog_api.get_catalog('foo', 'bar') +# self.assertDictEquals(catalog_ref, self.catalog_foobar) From c8ed28c7d302b4e0db2e36161e4ed74102964996 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 17:03:19 -0800 Subject: [PATCH 130/334] missed a file --- keystonelight/backends/sql/core.py | 2 + tests/test_backend.py | 91 ++++++++++++++++++++++++++++++ tests/test_backend_kvs.py | 4 +- tests/test_backend_sql.py | 4 +- 4 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 tests/test_backend.py diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py index 6bd9bcecec..a71d60efab 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystonelight/backends/sql/core.py @@ -187,6 +187,8 @@ class SqlIdentity(SqlBase): extras_ref = None if not user_ref or user_ref.get('password') != password: raise AssertionError('Invalid user / password') + + tenants = self.get_tenants_for_user(user_id) if tenant_id and tenant_id not in user_ref['tenants']: raise AssertionError('Invalid tenant') diff --git a/tests/test_backend.py b/tests/test_backend.py new file mode 100644 index 0000000000..31c37a2f70 --- /dev/null +++ b/tests/test_backend.py @@ -0,0 +1,91 @@ +class IdentityTests(object): + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') + + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) + + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(not extras_ref) + + def test_authenticate(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) + + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + self.assertDictEquals(user_ref, self.user_foo) + + def test_get_extras_bad_user(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(extras_ref is None) + + def test_get_extras_bad_tenant(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(extras_ref is None) + + def test_get_extras(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_role(self): + role_ref = self.identity_api.get_role( + role_id=self.role_keystone_admin['id']) + self.assertDictEquals(role_ref, self.role_keystone_admin) + + diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 089d088563..ca367c476e 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -4,11 +4,11 @@ from keystonelight import models from keystonelight import test from keystonelight.backends import kvs -import test_backends +import test_backend import default_fixtures -class KvsIdentity(test.TestCase, test_backends.IdentityTests): +class KvsIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(KvsIdentity, self).setUp() self.options = self.appconfig('default') diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index 1f64d898f5..bee4ec33fc 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -6,11 +6,11 @@ from keystonelight import test from keystonelight.backends import sql from keystonelight.backends.sql import migration -import test_backends +import test_backend import default_fixtures -class SqlIdentity(test.TestCase, test_backends.IdentityTests): +class SqlIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(SqlIdentity, self).setUp() self.options = self.appconfig('default') From 8fdcb69d4d6e155852c7d9fd35292b0e59f1469d Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 17:05:26 -0800 Subject: [PATCH 131/334] fix pep8 --- keystonelight/backends/sql/core.py | 6 ++++-- .../sql/migrate_repo/versions/001_add_initial_tables.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py index a71d60efab..060a98aebe 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystonelight/backends/sql/core.py @@ -15,6 +15,7 @@ from keystonelight import models Base = declarative.declarative_base() + # Special Fields class JsonBlob(sql_types.TypeDecorator): impl = sql.Text @@ -64,6 +65,7 @@ class DictBase(object): local.update(joined) return local.iteritems() + # Tables class User(Base, DictBase): __tablename__ = 'user' @@ -114,7 +116,7 @@ class UserTenantMembership(Base, DictBase): primary_key=True) - +# Backends class SqlBase(object): _MAKER = None _ENGINE = None @@ -189,7 +191,7 @@ class SqlIdentity(SqlBase): raise AssertionError('Invalid user / password') tenants = self.get_tenants_for_user(user_id) - if tenant_id and tenant_id not in user_ref['tenants']: + if tenant_id and tenant_id not in tenants: raise AssertionError('Invalid tenant') tenant_ref = self.get_tenant(tenant_id) diff --git a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py index e54106945a..ecfd6f507d 100644 --- a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py +++ b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py @@ -3,6 +3,7 @@ from migrate import * from keystonelight.backends import sql + def upgrade(migrate_engine): # Upgrade operations go here. Don't create your own engine; bind # migrate_engine to your metadata From 829a96b1e08585b77443fcc2317a773bfd097655 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 18:33:24 -0800 Subject: [PATCH 132/334] add nova's cfg framework --- keystonelight/cfg.py | 1126 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1126 insertions(+) create mode 100644 keystonelight/cfg.py diff --git a/keystonelight/cfg.py b/keystonelight/cfg.py new file mode 100644 index 0000000000..54940e7d90 --- /dev/null +++ b/keystonelight/cfg.py @@ -0,0 +1,1126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +r""" +Configuration options which may be set on the command line or in config files. + +The schema for each option is defined using the Opt sub-classes e.g. + + common_opts = [ + cfg.StrOpt('bind_host', + default='0.0.0.0', + help='IP address to listen on'), + cfg.IntOpt('bind_port', + default=9292, + help='Port number to listen on') + ] + +Options can be strings, integers, floats, booleans, lists or 'multi strings': + + enabled_apis_opt = \ + cfg.ListOpt('enabled_apis', + default=['ec2', 'osapi'], + help='List of APIs to enable by default') + + DEFAULT_EXTENSIONS = [ + 'nova.api.openstack.contrib.standard_extensions' + ] + osapi_extension_opt = \ + cfg.MultiStrOpt('osapi_extension', + default=DEFAULT_EXTENSIONS) + +Option schemas are registered with with the config manager at runtime, but +before the option is referenced: + + class ExtensionManager(object): + + enabled_apis_opt = cfg.ListOpt(...) + + def __init__(self, conf): + self.conf = conf + self.conf.register_opt(enabled_apis_opt) + ... + + def _load_extensions(self): + for ext_factory in self.conf.osapi_extension: + .... + +A common usage pattern is for each option schema to be defined in the module or +class which uses the option: + + opts = ... + + def add_common_opts(conf): + conf.register_opts(opts) + + def get_bind_host(conf): + return conf.bind_host + + def get_bind_port(conf): + return conf.bind_port + +An option may optionally be made available via the command line. Such options +must registered with the config manager before the command line is parsed (for +the purposes of --help and CLI arg validation): + + cli_opts = [ + cfg.BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output'), + cfg.BoolOpt('debug', + short='d', + default=False, + help='Print debugging output'), + ] + + def add_common_opts(conf): + conf.register_cli_opts(cli_opts) + +The config manager has a single CLI option defined by default, --config-file: + + class ConfigOpts(object): + + config_file_opt = \ + MultiStrOpt('config-file', + ... + + def __init__(self, ...): + ... + self.register_cli_opt(self.config_file_opt) + +Option values are parsed from any supplied config files using SafeConfigParser. +If none are specified, a default set is used e.g. glance-api.conf and +glance-common.conf: + + glance-api.conf: + [DEFAULT] + bind_port = 9292 + + glance-common.conf: + [DEFAULT] + bind_host = 0.0.0.0 + +Option values in config files override those on the command line. Config files +are parsed in order, with values in later files overriding those in earlier +files. + +The parsing of CLI args and config files is initiated by invoking the config +manager e.g. + + conf = ConfigOpts() + conf.register_opt(BoolOpt('verbose', ...)) + conf(sys.argv[1:]) + if conf.verbose: + ... + +Options can be registered as belonging to a group: + + rabbit_group = cfg.OptionGroup(name='rabbit', + title='RabbitMQ options') + + rabbit_host_opt = \ + cfg.StrOpt('host', + group='rabbit', + default='localhost', + help='IP/hostname to listen on'), + rabbit_port_opt = \ + cfg.IntOpt('port', + default=5672, + help='Port number to listen on') + rabbit_ssl_opt = \ + conf.BoolOpt('use_ssl', + default=False, + help='Whether to support SSL connections') + + def register_rabbit_opts(conf): + conf.register_group(rabbit_group) + # options can be registered under a group in any of these ways: + conf.register_opt(rabbit_host_opt) + conf.register_opt(rabbit_port_opt, group='rabbit') + conf.register_opt(rabbit_ssl_opt, group=rabbit_group) + +If no group is specified, options belong to the 'DEFAULT' section of config +files: + + glance-api.conf: + [DEFAULT] + bind_port = 9292 + ... + + [rabbit] + host = localhost + port = 5672 + use_ssl = False + userid = guest + password = guest + virtual_host = / + +Command-line options in a group are automatically prefixed with the group name: + + --rabbit-host localhost --rabbit-use-ssl False + +Option values in the default group are referenced as attributes/properties on +the config manager; groups are also attributes on the config manager, with +attributes for each of the options associated with the group: + + server.start(app, conf.bind_port, conf.bind_host, conf) + + self.connection = kombu.connection.BrokerConnection( + hostname=conf.rabbit.host, + port=conf.rabbit.port, + ...) + +Option values may reference other values using PEP 292 string substitution: + + opts = [ + cfg.StrOpt('state_path', + default=os.path.join(os.path.dirname(__file__), '../'), + help='Top-level directory for maintaining nova state'), + cfg.StrOpt('sqlite_db', + default='nova.sqlite', + help='file name for sqlite'), + cfg.StrOpt('sql_connection', + default='sqlite:///$state_path/$sqlite_db', + help='connection string for sql database'), + ] + +Note that interpolation can be avoided by using '$$'. +""" + +import sys +import ConfigParser +import copy +import optparse +import os +import string + + +class Error(Exception): + """Base class for cfg exceptions.""" + + def __init__(self, msg=None): + self.msg = msg + + def __str__(self): + return self.msg + + +class ArgsAlreadyParsedError(Error): + """Raised if a CLI opt is registered after parsing.""" + + def __str__(self): + ret = "arguments already parsed" + if self.msg: + ret += ": " + self.msg + return ret + + +class NoSuchOptError(Error): + """Raised if an opt which doesn't exist is referenced.""" + + def __init__(self, opt_name, group=None): + self.opt_name = opt_name + self.group = group + + def __str__(self): + if self.group is None: + return "no such option: %s" % self.opt_name + else: + return "no such option in group %s: %s" % (self.group.name, + self.opt_name) + + +class NoSuchGroupError(Error): + """Raised if a group which doesn't exist is referenced.""" + + def __init__(self, group_name): + self.group_name = group_name + + def __str__(self): + return "no such group: %s" % self.group_name + + +class DuplicateOptError(Error): + """Raised if multiple opts with the same name are registered.""" + + def __init__(self, opt_name): + self.opt_name = opt_name + + def __str__(self): + return "duplicate option: %s" % self.opt_name + + +class TemplateSubstitutionError(Error): + """Raised if an error occurs substituting a variable in an opt value.""" + + def __str__(self): + return "template substitution error: %s" % self.msg + + +class ConfigFilesNotFoundError(Error): + """Raised if one or more config files are not found.""" + + def __init__(self, config_files): + self.config_files = config_files + + def __str__(self): + return 'Failed to read some config files: %s' % \ + string.join(self.config_files, ',') + + +class ConfigFileParseError(Error): + """Raised if there is an error parsing a config file.""" + + def __init__(self, config_file, msg): + self.config_file = config_file + self.msg = msg + + def __str__(self): + return 'Failed to parse %s: %s' % (self.config_file, self.msg) + + +class ConfigFileValueError(Error): + """Raised if a config file value does not match its opt type.""" + pass + + +def find_config_files(project=None, prog=None): + """Return a list of default configuration files. + + We default to two config files: [${project}.conf, ${prog}.conf] + + And we look for those config files in the following directories: + + ~/.${project}/ + ~/ + /etc/${project}/ + /etc/ + + We return an absolute path for (at most) one of each the default config + files, for the topmost directory it exists in. + + For example, if project=foo, prog=bar and /etc/foo/foo.conf, /etc/bar.conf + and ~/.foo/bar.conf all exist, then we return ['/etc/foo/foo.conf', + '~/.foo/bar.conf'] + + If no project name is supplied, we only look for ${prog.conf}. + + :param project: an optional project name + :param prog: the program name, defaulting to the basename of sys.argv[0] + """ + if prog is None: + prog = os.path.basename(sys.argv[0]) + + fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) + + cfg_dirs = [ + fix_path(os.path.join('~', '.' + project)) if project else None, + fix_path('~'), + os.path.join('/etc', project) if project else None, + '/etc' + ] + cfg_dirs = filter(bool, cfg_dirs) + + def search_dirs(dirs, basename): + for d in dirs: + path = os.path.join(d, basename) + if os.path.exists(path): + return path + + config_files = [] + if project: + config_files.append(search_dirs(cfg_dirs, '%s.conf' % project)) + config_files.append(search_dirs(cfg_dirs, '%s.conf' % prog)) + + return filter(bool, config_files) + + +def _is_opt_registered(opts, opt): + """Check whether an opt with the same name is already registered. + + The same opt may be registered multiple times, with only the first + registration having any effect. However, it is an error to attempt + to register a different opt with the same name. + + :param opts: the set of opts already registered + :param opt: the opt to be registered + :returns: True if the opt was previously registered, False otherwise + :raises: DuplicateOptError if a naming conflict is detected + """ + if opt.dest in opts: + if opts[opt.dest]['opt'] is not opt: + raise DuplicateOptError(opt.name) + return True + else: + return False + + +class Opt(object): + + """Base class for all configuration options. + + An Opt object has no public methods, but has a number of public string + properties: + + name: + the name of the option, which may include hyphens + dest: + the (hyphen-less) ConfigOpts property which contains the option value + short: + a single character CLI option name + default: + the default value of the option + metavar: + the name shown as the argument to a CLI option in --help output + help: + an string explaining how the options value is used + """ + + def __init__(self, name, dest=None, short=None, + default=None, metavar=None, help=None): + """Construct an Opt object. + + The only required parameter is the option's name. However, it is + common to also supply a default and help string for all options. + + :param name: the option's name + :param dest: the name of the corresponding ConfigOpts property + :param short: a single character CLI option name + :param default: the default value of the option + :param metavar: the option argument to show in --help + :param help: an explanation of how the option is used + """ + self.name = name + if dest is None: + self.dest = self.name.replace('-', '_') + else: + self.dest = dest + self.short = short + self.default = default + self.metavar = metavar + self.help = help + + def _get_from_config_parser(self, cparser, section): + """Retrieves the option value from a ConfigParser object. + + This is the method ConfigOpts uses to look up the option value from + config files. Most opt types override this method in order to perform + type appropriate conversion of the returned value. + + :param cparser: a ConfigParser object + :param section: a section name + """ + return cparser.get(section, self.dest) + + def _add_to_cli(self, parser, group=None): + """Makes the option available in the command line interface. + + This is the method ConfigOpts uses to add the opt to the CLI interface + as appropriate for the opt type. Some opt types may extend this method, + others may just extend the helper methods it uses. + + :param parser: the CLI option parser + :param group: an optional OptGroup object + """ + container = self._get_optparse_container(parser, group) + kwargs = self._get_optparse_kwargs(group) + prefix = self._get_optparse_prefix('', group) + self._add_to_optparse(container, self.name, self.short, kwargs, prefix) + + def _add_to_optparse(self, container, name, short, kwargs, prefix=''): + """Add an option to an optparse parser or group. + + :param container: an optparse.OptionContainer object + :param name: the opt name + :param short: the short opt name + :param kwargs: the keyword arguments for add_option() + :param prefix: an optional prefix to prepend to the opt name + :raises: DuplicateOptError if a naming confict is detected + """ + args = ['--' + prefix + name] + if short: + args += ['-' + short] + for a in args: + if container.has_option(a): + raise DuplicateOptError(a) + container.add_option(*args, **kwargs) + + def _get_optparse_container(self, parser, group): + """Returns an optparse.OptionContainer. + + :param parser: an optparse.OptionParser + :param group: an (optional) OptGroup object + :returns: an optparse.OptionGroup if a group is given, else the parser + """ + if group is not None: + return group._get_optparse_group(parser) + else: + return parser + + def _get_optparse_kwargs(self, group, **kwargs): + """Build a dict of keyword arguments for optparse's add_option(). + + Most opt types extend this method to customize the behaviour of the + options added to optparse. + + :param group: an optional group + :param kwargs: optional keyword arguments to add to + :returns: a dict of keyword arguments + """ + dest = self.dest + if group is not None: + dest = group.name + '_' + dest + kwargs.update({ + 'dest': dest, + 'metavar': self.metavar, + 'help': self.help, + }) + return kwargs + + def _get_optparse_prefix(self, prefix, group): + """Build a prefix for the CLI option name, if required. + + CLI options in a group are prefixed with the group's name in order + to avoid conflicts between similarly named options in different + groups. + + :param prefix: an existing prefix to append to (e.g. 'no' or '') + :param group: an optional OptGroup object + :returns: a CLI option prefix including the group name, if appropriate + """ + if group is not None: + return group.name + '-' + prefix + else: + return prefix + + +class StrOpt(Opt): + """ + String opts do not have their values transformed and are returned as + str objects. + """ + pass + + +class BoolOpt(Opt): + + """ + Bool opts are set to True or False on the command line using --optname or + --noopttname respectively. + + In config files, boolean values are case insensitive and can be set using + 1/0, yes/no, true/false or on/off. + """ + + def _get_from_config_parser(self, cparser, section): + """Retrieve the opt value as a boolean from ConfigParser.""" + return cparser.getboolean(section, self.dest) + + def _add_to_cli(self, parser, group=None): + """Extends the base class method to add the --nooptname option.""" + super(BoolOpt, self)._add_to_cli(parser, group) + self._add_inverse_to_optparse(parser, group) + + def _add_inverse_to_optparse(self, parser, group): + """Add the --nooptname option to the option parser.""" + container = self._get_optparse_container(parser, group) + kwargs = self._get_optparse_kwargs(group, action='store_false') + prefix = self._get_optparse_prefix('no', group) + self._add_to_optparse(container, self.name, None, kwargs, prefix) + + def _get_optparse_kwargs(self, group, action='store_true', **kwargs): + """Extends the base optparse keyword dict for boolean options.""" + return super(BoolOpt, + self)._get_optparse_kwargs(group, action=action, **kwargs) + + +class IntOpt(Opt): + + """Int opt values are converted to integers using the int() builtin.""" + + def _get_from_config_parser(self, cparser, section): + """Retrieve the opt value as a integer from ConfigParser.""" + return cparser.getint(section, self.dest) + + def _get_optparse_kwargs(self, group, **kwargs): + """Extends the base optparse keyword dict for integer options.""" + return super(IntOpt, + self)._get_optparse_kwargs(group, type='int', **kwargs) + + +class FloatOpt(Opt): + + """Float opt values are converted to floats using the float() builtin.""" + + def _get_from_config_parser(self, cparser, section): + """Retrieve the opt value as a float from ConfigParser.""" + return cparser.getfloat(section, self.dest) + + def _get_optparse_kwargs(self, group, **kwargs): + """Extends the base optparse keyword dict for float options.""" + return super(FloatOpt, + self)._get_optparse_kwargs(group, type='float', **kwargs) + + +class ListOpt(Opt): + + """ + List opt values are simple string values separated by commas. The opt value + is a list containing these strings. + """ + + def _get_from_config_parser(self, cparser, section): + """Retrieve the opt value as a list from ConfigParser.""" + return cparser.get(section, self.dest).split(',') + + def _get_optparse_kwargs(self, group, **kwargs): + """Extends the base optparse keyword dict for list options.""" + return super(ListOpt, + self)._get_optparse_kwargs(group, + type='string', + action='callback', + callback=self._parse_list, + **kwargs) + + def _parse_list(self, option, opt, value, parser): + """An optparse callback for parsing an option value into a list.""" + setattr(parser.values, self.dest, value.split(',')) + + +class MultiStrOpt(Opt): + + """ + Multistr opt values are string opts which may be specified multiple times. + The opt value is a list containing all the string values specified. + """ + + def _get_from_config_parser(self, cparser, section): + """Retrieve the opt value as a multistr from ConfigParser.""" + # FIXME(markmc): values spread across the CLI and multiple + # config files should be appended + value = \ + super(MultiStrOpt, self)._get_from_config_parser(cparser, section) + return value if value is None else [value] + + def _get_optparse_kwargs(self, group, **kwargs): + """Extends the base optparse keyword dict for multi str options.""" + return super(MultiStrOpt, + self)._get_optparse_kwargs(group, action='append') + + +class OptGroup(object): + + """ + Represents a group of opts. + + CLI opts in the group are automatically prefixed with the group name. + + Each group corresponds to a section in config files. + + An OptGroup object has no public methods, but has a number of public string + properties: + + name: + the name of the group + title: + the group title as displayed in --help + help: + the group description as displayed in --help + """ + + def __init__(self, name, title=None, help=None): + """Constructs an OptGroup object. + + :param name: the group name + :param title: the group title for --help + :param help: the group description for --help + """ + self.name = name + if title is None: + self.title = "%s options" % title + else: + self.title = title + self.help = help + + self._opts = {} # dict of dicts of {opt:, override:, default:) + self._optparse_group = None + + def _register_opt(self, opt): + """Add an opt to this group. + + :param opt: an Opt object + :returns: False if previously registered, True otherwise + :raises: DuplicateOptError if a naming conflict is detected + """ + if _is_opt_registered(self._opts, opt): + return False + + self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None} + + return True + + def _get_optparse_group(self, parser): + """Build an optparse.OptionGroup for this group.""" + if self._optparse_group is None: + self._optparse_group = \ + optparse.OptionGroup(parser, self.title, self.help) + return self._optparse_group + + +class ConfigOpts(object): + + """ + Config options which may be set on the command line or in config files. + + ConfigOpts is a configuration option manager with APIs for registering + option schemas, grouping options, parsing option values and retrieving + the values of options. + """ + + def __init__(self, + project=None, + prog=None, + version=None, + usage=None, + default_config_files=None): + """Construct a ConfigOpts object. + + Automatically registers the --config-file option with either a supplied + list of default config files, or a list from find_config_files(). + + :param project: the toplevel project name, used to locate config files + :param prog: the name of the program (defaults to sys.argv[0] basename) + :param version: the program version (for --version) + :param usage: a usage string (%prog will be expanded) + :param default_config_files: config files to use by default + """ + if prog is None: + prog = os.path.basename(sys.argv[0]) + + if default_config_files is None: + default_config_files = find_config_files(project, prog) + + self.project = project + self.prog = prog + self.version = version + self.usage = usage + self.default_config_files = default_config_files + + self._opts = {} # dict of dicts of (opt:, override:, default:) + self._groups = {} + + self._args = None + self._cli_values = {} + + self._oparser = optparse.OptionParser(prog=self.prog, + version=self.version, + usage=self.usage) + self._cparser = None + + self.register_cli_opt(\ + MultiStrOpt('config-file', + default=self.default_config_files, + metavar='PATH', + help='Path to a config file to use. Multiple config ' + 'files can be specified, with values in later ' + 'files taking precedence. The default files used ' + 'are: %s' % (self.default_config_files, ))) + + def __call__(self, args=None): + """Parse command line arguments and config files. + + Calling a ConfigOpts object causes the supplied command line arguments + and config files to be parsed, causing opt values to be made available + as attributes of the object. + + The object may be called multiple times, each time causing the previous + set of values to be overwritten. + + :params args: command line arguments (defaults to sys.argv[1:]) + :returns: the list of arguments left over after parsing options + :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError + """ + self.reset() + + self._args = args + + (values, args) = self._oparser.parse_args(self._args) + + self._cli_values = vars(values) + + if self.config_file: + self._parse_config_files(self.config_file) + + return args + + def __getattr__(self, name): + """Look up an option value and perform string substitution. + + :param name: the opt name (or 'dest', more precisely) + :returns: the option value (after string subsititution) or a GroupAttr + :raises: NoSuchOptError,ConfigFileValueError,TemplateSubstitutionError + """ + return self._substitute(self._get(name)) + + def reset(self): + """Reset the state of the object to before it was called.""" + self._args = None + self._cli_values = None + self._cparser = None + + def register_opt(self, opt, group=None): + """Register an option schema. + + Registering an option schema makes any option value which is previously + or subsequently parsed from the command line or config files available + as an attribute of this object. + + :param opt: an instance of an Opt sub-class + :param group: an optional OptGroup object or group name + :return: False if the opt was already register, True otherwise + :raises: DuplicateOptError + """ + if group is not None: + return self._get_group(group)._register_opt(opt) + + if _is_opt_registered(self._opts, opt): + return False + + self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None} + + return True + + def register_opts(self, opts, group=None): + """Register multiple option schemas at once.""" + for opt in opts: + self.register_opt(opt, group) + + def register_cli_opt(self, opt, group=None): + """Register a CLI option schema. + + CLI option schemas must be registered before the command line and + config files are parsed. This is to ensure that all CLI options are + show in --help and option validation works as expected. + + :param opt: an instance of an Opt sub-class + :param group: an optional OptGroup object or group name + :return: False if the opt was already register, True otherwise + :raises: DuplicateOptError, ArgsAlreadyParsedError + """ + if self._args is not None: + raise ArgsAlreadyParsedError("cannot register CLI option") + + if not self.register_opt(opt, group): + return False + + if group is not None: + group = self._get_group(group) + + opt._add_to_cli(self._oparser, group) + + return True + + def register_cli_opts(self, opts, group=None): + """Register multiple CLI option schemas at once.""" + for opt in opts: + self.register_cli_opt(opt, group) + + def register_group(self, group): + """Register an option group. + + An option group must be registered before options can be registered + with the group. + + :param group: an OptGroup object + """ + if group.name in self._groups: + return + + self._groups[group.name] = copy.copy(group) + + def set_override(self, name, override, group=None): + """Override an opt value. + + Override the command line, config file and default values of a + given option. + + :param name: the name/dest of the opt + :param override: the override value + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + opt_info = self._get_opt_info(name, group) + opt_info['override'] = override + + def set_default(self, name, default, group=None): + """Override an opt's default value. + + Override the default value of given option. A command line or + config file value will still take precedence over this default. + + :param name: the name/dest of the opt + :param default: the default value + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + opt_info = self._get_opt_info(name, group) + opt_info['default'] = default + + def log_opt_values(self, logger, lvl): + """Log the value of all registered opts. + + It's often useful for an app to log its configuration to a log file at + startup for debugging. This method dumps to the entire config state to + the supplied logger at a given log level. + + :param logger: a logging.Logger object + :param lvl: the log level (e.g. logging.DEBUG) arg to logger.log() + """ + logger.log(lvl, "*" * 80) + logger.log(lvl, "Configuration options gathered from:") + logger.log(lvl, "command line args: %s", self._args) + logger.log(lvl, "config files: %s", self.config_file) + logger.log(lvl, "=" * 80) + + for opt_name in sorted(self._opts): + logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name)) + + for group_name in self._groups: + group_attr = self.GroupAttr(self, group_name) + for opt_name in sorted(self._groups[group_name]._opts): + logger.log(lvl, "%-30s = %s", + "%s.%s" % (group_name, opt_name), + getattr(group_attr, opt_name)) + + logger.log(lvl, "*" * 80) + + def print_usage(self, file=None): + """Print the usage message for the current program.""" + self._oparser.print_usage(file) + + def _get(self, name, group=None): + """Look up an option value. + + :param name: the opt name (or 'dest', more precisely) + :param group: an option OptGroup + :returns: the option value, or a GroupAttr object + :raises: NoSuchOptError, NoSuchGroupError, ConfigFileValueError, + TemplateSubstitutionError + """ + if group is None and name in self._groups: + return self.GroupAttr(self, name) + + if group is not None: + group = self._get_group(group) + + info = self._get_opt_info(name, group) + default, opt, override = map(lambda k: info[k], sorted(info.keys())) + + if override is not None: + return override + + if self._cparser is not None: + section = group.name if group is not None else 'DEFAULT' + try: + return opt._get_from_config_parser(self._cparser, section) + except (ConfigParser.NoOptionError, + ConfigParser.NoSectionError): + pass + except ValueError, ve: + raise ConfigFileValueError(str(ve)) + + name = name if group is None else group.name + '_' + name + value = self._cli_values.get(name, None) + if value is not None: + return value + + if default is not None: + return default + + return opt.default + + def _substitute(self, value): + """Perform string template substitution. + + Substititue any template variables (e.g. $foo, ${bar}) in the supplied + string value(s) with opt values. + + :param value: the string value, or list of string values + :returns: the substituted string(s) + """ + if isinstance(value, list): + return [self._substitute(i) for i in value] + elif isinstance(value, str): + tmpl = string.Template(value) + return tmpl.safe_substitute(self.StrSubWrapper(self)) + else: + return value + + def _get_group(self, group_or_name): + """Looks up a OptGroup object. + + Helper function to return an OptGroup given a parameter which can + either be the group's name or an OptGroup object. + + The OptGroup object returned is from the internal dict of OptGroup + objects, which will be a copy of any OptGroup object that users of + the API have access to. + + :param group_or_name: the group's name or the OptGroup object itself + :raises: NoSuchGroupError + """ + if isinstance(group_or_name, OptGroup): + group_name = group_or_name.name + else: + group_name = group_or_name + + if not group_name in self._groups: + raise NoSuchGroupError(group_name) + + return self._groups[group_name] + + def _get_opt_info(self, opt_name, group=None): + """Return the (opt, override, default) dict for an opt. + + :param opt_name: an opt name/dest + :param group: an optional group name or OptGroup object + :raises: NoSuchOptError, NoSuchGroupError + """ + if group is None: + opts = self._opts + else: + group = self._get_group(group) + opts = group._opts + + if not opt_name in opts: + raise NoSuchOptError(opt_name, group) + + return opts[opt_name] + + def _parse_config_files(self, config_files): + """Parse the supplied configuration files. + + :raises: ConfigFilesNotFoundError, ConfigFileParseError + """ + self._cparser = ConfigParser.SafeConfigParser() + + try: + read_ok = self._cparser.read(config_files) + except ConfigParser.ParsingError, cpe: + raise ConfigFileParseError(cpe.filename, cpe.message) + + if read_ok != config_files: + not_read_ok = filter(lambda f: f not in read_ok, config_files) + raise ConfigFilesNotFoundError(not_read_ok) + + class GroupAttr(object): + + """ + A helper class representing the option values of a group as attributes. + """ + + def __init__(self, conf, group): + """Construct a GroupAttr object. + + :param conf: a ConfigOpts object + :param group: a group name or OptGroup object + """ + self.conf = conf + self.group = group + + def __getattr__(self, name): + """Look up an option value and perform template substitution.""" + return self.conf._substitute(self.conf._get(name, self.group)) + + class StrSubWrapper(object): + + """ + A helper class exposing opt values as a dict for string substitution. + """ + + def __init__(self, conf): + """Construct a StrSubWrapper object. + + :param conf: a ConfigOpts object + """ + self.conf = conf + + def __getitem__(self, key): + """Look up an opt value from the ConfigOpts object. + + :param key: an opt name + :returns: an opt value + :raises: TemplateSubstitutionError if attribute is a group + """ + value = getattr(self.conf, key) + if isinstance(value, self.conf.GroupAttr): + raise TemplateSubstitutionError( + 'substituting group %s not supported' % key) + return value + + +class CommonConfigOpts(ConfigOpts): + + DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" + DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + + common_cli_opts = [ + BoolOpt('debug', + short='d', + default=False, + help='Print debugging output'), + BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output'), + ] + + logging_cli_opts = [ + StrOpt('log-config', + metavar='PATH', + help='If this option is specified, the logging configuration ' + 'file specified is used and overrides any other logging ' + 'options specified. Please see the Python logging module ' + 'documentation for details on logging configuration ' + 'files.'), + StrOpt('log-format', + default=DEFAULT_LOG_FORMAT, + metavar='FORMAT', + help='A logging.Formatter log message format string which may ' + 'use any of the available logging.LogRecord attributes. ' + 'Default: %default'), + StrOpt('log-date-format', + default=DEFAULT_LOG_DATE_FORMAT, + metavar='DATE_FORMAT', + help='Format string for %(asctime)s in log records. ' + 'Default: %default'), + StrOpt('log-file', + metavar='PATH', + help='(Optional) Name of log file to output to. ' + 'If not set, logging will go to stdout.'), + StrOpt('log-dir', + help='(Optional) The directory to keep log files in ' + '(will be prepended to --logfile)'), + BoolOpt('use-syslog', + default=False, + help='Use syslog for logging.'), + ] + + def __init__(self, **kwargs): + super(CommonConfigOpts, self).__init__(**kwargs) + self.register_cli_opts(self.common_cli_opts) + self.register_cli_opts(self.logging_cli_opts) From feadf7576058f47f2bb1f0bc097d45101e8c1562 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 21:00:41 -0800 Subject: [PATCH 133/334] config system overhaul we're now based on nova's config system which mainly means that you will have to call config.CONF with a config file at some point during tests or running an executable --- keystonelight/backends/kvs.py | 8 +- keystonelight/backends/policy.py | 6 -- keystonelight/backends/sql/core.py | 32 ++------ keystonelight/backends/sql/migration.py | 24 +++--- keystonelight/backends/templated.py | 25 +++--- keystonelight/catalog.py | 10 ++- keystonelight/config.py | 49 +++++++++++ keystonelight/identity.py | 11 +-- keystonelight/keystone_compat.py | 103 +++++++++++------------- keystonelight/middleware.py | 6 +- keystonelight/policy.py | 10 ++- keystonelight/service.py | 25 +++--- keystonelight/test.py | 58 +++---------- keystonelight/token.py | 10 ++- keystonelight/wsgi.py | 12 ++- tests/default.conf | 42 +++++----- tests/default_catalog.templates | 12 +++ tests/keystone_compat_diablo.conf | 17 +++- tests/keystoneclient_compat_master.conf | 34 ++++---- tests/test_backend_kvs.py | 9 +-- tests/test_backend_sql.py | 20 +++-- tests/test_identity_api.py | 12 ++- tests/test_keystoneclient_compat.py | 9 ++- tests/test_legacy_compat.py | 35 ++++---- tests/test_novaclient_compat.py | 6 +- 25 files changed, 304 insertions(+), 281 deletions(-) create mode 100644 keystonelight/config.py create mode 100644 tests/default_catalog.templates diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 5bb8eddd27..4394f34776 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -10,7 +10,7 @@ INMEMDB = DictKvs() class KvsIdentity(object): - def __init__(self, options, db=None): + def __init__(self, db=None): if db is None: db = INMEMDB elif type(db) is type({}): @@ -189,7 +189,7 @@ class KvsIdentity(object): class KvsToken(object): - def __init__(self, options, db=None): + def __init__(self, db=None): if db is None: db = INMEMDB elif type(db) is type({}): @@ -209,7 +209,7 @@ class KvsToken(object): class KvsCatalog(object): - def __init__(self, options, db=None): + def __init__(self, db=None): if db is None: db = INMEMDB elif type(db) is type({}): @@ -251,7 +251,7 @@ class KvsCatalog(object): class KvsPolicy(object): - def __init__(self, options, db=None): + def __init__(self, db=None): if db is None: db = INMEMDB elif type(db) is type({}): diff --git a/keystonelight/backends/policy.py b/keystonelight/backends/policy.py index 6bdcacb80b..bb33548db1 100644 --- a/keystonelight/backends/policy.py +++ b/keystonelight/backends/policy.py @@ -2,17 +2,11 @@ import logging class TrivialTrue(object): - def __init__(self, options): - self.options = options - def can_haz(self, target, credentials): return True class SimpleMatch(object): - def __init__(self, options): - self.options = options - def can_haz(self, target, credentials): """Check whether key-values in target are present in credentials.""" # TODO(termie): handle ANDs, probably by providing a tuple instead of a diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py index 060a98aebe..211d9ca261 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystonelight/backends/sql/core.py @@ -10,9 +10,13 @@ import sqlalchemy.orm import sqlalchemy.pool import sqlalchemy.engine.url +from keystonelight import config from keystonelight import models +CONF = config.CONF + + Base = declarative.declarative_base() @@ -121,9 +125,6 @@ class SqlBase(object): _MAKER = None _ENGINE = None - def __init__(self, options): - self.options = options - def get_session(self, autocommit=True, expire_on_commit=False): """Return a SQLAlchemy session.""" if self._MAKER is None or self._ENGINE is None: @@ -138,36 +139,17 @@ class SqlBase(object): def get_engine(self): """Return a SQLAlchemy engine.""" - connection_dict = sqlalchemy.engine.url.make_url( - self.options.get('sql_connection')) + connection_dict = sqlalchemy.engine.url.make_url(CONF.sql.connection) engine_args = { - "pool_recycle": self.options.get('sql_idle_timeout'), + "pool_recycle": CONF.sql.idle_timeout, "echo": False, } if "sqlite" in connection_dict.drivername: engine_args["poolclass"] = sqlalchemy.pool.NullPool - #elif MySQLdb and "mysql" in connection_dict.drivername: - # LOG.info(_("Using mysql/eventlet db_pool.")) - # # MySQLdb won't accept 'None' in the password field - # password = connection_dict.password or '' - # pool_args = { - # "db": connection_dict.database, - # "passwd": password, - # "host": connection_dict.host, - # "user": connection_dict.username, - # "min_size": self.options.get('sql_min_pool_size'), - # "max_size": self.options.get('sql_max_pool_size'), - # "max_idle": self.options.get('sql_idle_timeout'), - # } - # creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) - # engine_args["pool_size"] = self.options.get('sql_max_pool_size') - # engine_args["pool_timeout"] = self.options('sql_pool_timeout') - # engine_args["creator"] = creator.create - return sql.create_engine(self.options.get('sql_connection'), - **engine_args) + return sql.create_engine(CONF.sql.connection, **engine_args) def get_maker(self, engine, autocommit=True, expire_on_commit=False): """Return a SQLAlchemy sessionmaker using the given engine.""" diff --git a/keystonelight/backends/sql/migration.py b/keystonelight/backends/sql/migration.py index 1a4794b7e8..106bcea7ee 100644 --- a/keystonelight/backends/sql/migration.py +++ b/keystonelight/backends/sql/migration.py @@ -22,6 +22,12 @@ import sys import sqlalchemy from migrate.versioning import api as versioning_api +from keystonelight import config + + +CONF = config.CONF + + try: from migrate.versioning import exceptions as versioning_exceptions except ImportError: @@ -33,36 +39,36 @@ except ImportError: sys.exit("python-migrate is not installed. Exiting.") -def db_sync(options, version=None): +def db_sync(version=None): if version is not None: try: version = int(version) except ValueError: raise Exception("version should be an integer") - current_version = db_version(options) + current_version = db_version() repo_path = _find_migrate_repo() if version is None or version > current_version: return versioning_api.upgrade( - options.get('sql_connection'), repo_path, version) + CONF.sql.connection, repo_path, version) else: return versioning_api.downgrade( - options.get('sql_connection'), repo_path, version) + CONF.sql.connection, repo_path, version) -def db_version(options): +def db_version(): repo_path = _find_migrate_repo() try: return versioning_api.db_version( - options.get('sql_connection'), repo_path) + CONF.sql.connection, repo_path) except versioning_exceptions.DatabaseNotControlledError: - return db_version_control(options, 0) + return db_version_control(0) -def db_version_control(options, version=None): +def db_version_control(version=None): repo_path = _find_migrate_repo() versioning_api.version_control( - options.get('sql_connection'), repo_path, version) + CONF.sql.connection, repo_path, version) return version diff --git a/keystonelight/backends/templated.py b/keystonelight/backends/templated.py index 4ba5575851..155839b2be 100644 --- a/keystonelight/backends/templated.py +++ b/keystonelight/backends/templated.py @@ -1,7 +1,12 @@ +from keystonelight import config from keystonelight import logging from keystonelight.backends import kvs +CONF = config.CONF +config.register_str('template_file', group='catalog') + + class TemplatedCatalog(kvs.KvsCatalog): """A backend that generates endpoints for the Catalog based on templates. @@ -16,7 +21,7 @@ class TemplatedCatalog(kvs.KvsCatalog): http://localhost:$(public_port)s/ - When expanding the template it will pass in a dict made up of the options + When expanding the template it will pass in a dict made up of the conf instance plus a few additional key-values, notably tenant_id and user_id. It does not care what the keys and values are but it is worth noting that @@ -31,19 +36,21 @@ class TemplatedCatalog(kvs.KvsCatalog): """ - def __init__(self, options, templates=None): - self.options = options - + def __init__(self, templates=None): if templates: self.templates = templates else: - self._load_templates(options) + self._load_templates(CONF.catalog.template_file) - super(TemplatedCatalog, self).__init__(options) + super(TemplatedCatalog, self).__init__() - def _load_templates(self, options): + def _load_templates(self, template_file): o = {} - for k, v in options.iteritems(): + for line in open(template_file): + if ' = ' not in line: + continue + + k, v = line.split(' = ') if not k.startswith('catalog.'): continue @@ -64,7 +71,7 @@ class TemplatedCatalog(kvs.KvsCatalog): self.templates = o def get_catalog(self, user_id, tenant_id, extras=None): - d = self.options.copy() + d = dict(CONF.iteritems()) d.update({'tenant_id': tenant_id, 'user_id': user_id}) diff --git a/keystonelight/catalog.py b/keystonelight/catalog.py index 382b00ffaa..b4d472732e 100644 --- a/keystonelight/catalog.py +++ b/keystonelight/catalog.py @@ -2,14 +2,16 @@ # the catalog interfaces +from keystonelight import config from keystonelight import utils +CONF = config.CONF + + class Manager(object): - def __init__(self, options): - self.options = options - self.driver = utils.import_object(options['catalog_driver'], - options=options) + def __init__(self): + self.driver = utils.import_object(CONF.catalog.driver) def get_catalog(self, context, user_id, tenant_id, extras=None): """Return info for a catalog if it is valid.""" diff --git a/keystonelight/config.py b/keystonelight/config.py new file mode 100644 index 0000000000..03d6a24b81 --- /dev/null +++ b/keystonelight/config.py @@ -0,0 +1,49 @@ + + +from keystonelight import cfg + + +class Config(cfg.ConfigOpts): + def __call__(self, config_files=None, *args, **kw): + if config_files is None: + config_files = [] + self.config_file = config_files + super(Config, self).__call__(*args, **kw) + + def __getitem__(self, key, default=None): + return getattr(self, key, default) + + def __setitem__(self, key, value): + return setattr(self, key, value) + + def iteritems(self): + for k in self._opts: + yield (k, getattr(self, k)) + + +def register_str(*args, **kw): + group = kw.pop('group', None) + if group: + CONF.register_group(cfg.OptGroup(name=group)) + return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + + +CONF = Config() + + +register_str('admin_token', default='ADMIN') +register_str('compute_port') +register_str('admin_port') +register_str('public_port') + +# sql options +register_str('connection', group='sql') +register_str('idle_timeout', group='sql') +register_str('min_pool_size', group='sql') +register_str('maz_pool_size', group='sql') +register_str('pool_timeout', group='sql') + +register_str('driver', group='catalog') +register_str('driver', group='identity') +register_str('driver', group='policy') +register_str('driver', group='token') diff --git a/keystonelight/identity.py b/keystonelight/identity.py index 52959ef979..93e8d309a5 100644 --- a/keystonelight/identity.py +++ b/keystonelight/identity.py @@ -2,15 +2,16 @@ # backends will make use of them to return something that conforms to their # apis - +from keystonelight import config from keystonelight import utils +CONF = config.CONF + + class Manager(object): - def __init__(self, options): - self.driver = utils.import_object(options['identity_driver'], - options=options) - self.options = options + def __init__(self): + self.driver = utils.import_object(CONF.identity.driver) def authenticate(self, context, **kwargs): """Passthru authentication to the identity driver. diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index 0fd998e196..b720477ca9 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -18,13 +18,11 @@ from keystonelight import wsgi class KeystoneAdminRouter(wsgi.Router): - def __init__(self, options): - self.options = options - + def __init__(self): mapper = routes.Mapper() # Token Operations - auth_controller = KeystoneTokenController(self.options) + auth_controller = KeystoneTokenController() mapper.connect('/tokens', controller=auth_controller, action='authenticate', @@ -39,7 +37,7 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # Tenant Operations - tenant_controller = KeystoneTenantController(self.options) + tenant_controller = KeystoneTenantController() mapper.connect('/tenants', controller=tenant_controller, action='get_tenants_for_token', @@ -50,14 +48,14 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # User Operations - user_controller = KeystoneUserController(self.options) + user_controller = KeystoneUserController() mapper.connect('/users/{user_id}', controller=user_controller, action='get_user', conditions=dict(method=['GET'])) # Role Operations - roles_controller = KeystoneRoleController(self.options) + roles_controller = KeystoneRoleController() mapper.connect('/tenants/{tenant_id}/users/{user_id}/roles', controller=roles_controller, action='get_user_roles', @@ -68,13 +66,13 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # Miscellaneous Operations - version_controller = KeystoneVersionController(self.options) + version_controller = KeystoneVersionController() mapper.connect('/', controller=version_controller, action='get_version_info', module='admin/version', conditions=dict(method=['GET'])) - extensions_controller = KeystoneExtensionsController(self.options) + extensions_controller = KeystoneExtensionsController() mapper.connect('/extensions', controller=extensions_controller, action='get_extensions_info', @@ -84,12 +82,11 @@ class KeystoneAdminRouter(wsgi.Router): class KeystoneServiceRouter(wsgi.Router): - def __init__(self, options): - self.options = options + def __init__(self): mapper = routes.Mapper() # Token Operations - auth_controller = KeystoneTokenController(self.options) + auth_controller = KeystoneTokenController() mapper.connect('/tokens', controller=auth_controller, action='authenticate', @@ -100,21 +97,21 @@ class KeystoneServiceRouter(wsgi.Router): conditions=dict(methods=['POST'])) # Tenant Operations - tenant_controller = KeystoneTenantController(self.options) + tenant_controller = KeystoneTenantController() mapper.connect('/tenants', controller=tenant_controller, action='get_tenants_for_token', conditions=dict(methods=['GET'])) # Miscellaneous - version_controller = KeystoneVersionController(self.options) + version_controller = KeystoneVersionController() mapper.connect('/', controller=version_controller, action='get_version_info', module='service/version', conditions=dict(method=['GET'])) - extensions_controller = KeystoneExtensionsController(self.options) + extensions_controller = KeystoneExtensionsController() mapper.connect('/extensions', controller=extensions_controller, action='get_extensions_info', @@ -130,13 +127,12 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): """ - def __init__(self, application, options): - self.options = options + def __init__(self, application): mapper = routes.Mapper() - tenant_controller = KeystoneTenantController(self.options) - user_controller = KeystoneUserController(self.options) - role_controller = KeystoneRoleController(self.options) - service_controller = KeystoneServiceController(self.options) + tenant_controller = KeystoneTenantController() + user_controller = KeystoneUserController() + role_controller = KeystoneRoleController() + service_controller = KeystoneServiceController() # Tenant Operations mapper.connect("/tenants", controller=tenant_controller, @@ -270,16 +266,15 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): conditions=dict(method=["DELETE"])) super(KeystoneAdminCrudExtension, self).__init__( - application, options, mapper) + application, mapper) class KeystoneTokenController(service.BaseApplication): - def __init__(self, options): - self.options = options - self.catalog_api = catalog.Manager(options) - self.identity_api = identity.Manager(options) - self.token_api = token.Manager(options) - self.policy_api = policy.Manager(options) + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() super(KeystoneTokenController, self).__init__() def authenticate(self, context, auth=None): @@ -496,11 +491,10 @@ class KeystoneTokenController(service.BaseApplication): class KeystoneTenantController(service.BaseApplication): - def __init__(self, options): - self.options = options - self.identity_api = identity.Manager(options) - self.policy_api = policy.Manager(options) - self.token_api = token.Manager(options) + def __init__(self): + self.identity_api = identity.Manager() + self.policy_api = policy.Manager() + self.token_api = token.Manager() super(KeystoneTenantController, self).__init__() def get_tenants_for_token(self, context, **kw): @@ -578,12 +572,11 @@ class KeystoneTenantController(service.BaseApplication): class KeystoneUserController(service.BaseApplication): - def __init__(self, options): - self.options = options - self.catalog_api = catalog.Manager(options) - self.identity_api = identity.Manager(options) - self.token_api = token.Manager(options) - self.policy_api = policy.Manager(options) + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.policy_api = policy.Manager() + self.token_api = token.Manager() super(KeystoneUserController, self).__init__() def get_user(self, context, user_id): @@ -644,12 +637,11 @@ class KeystoneUserController(service.BaseApplication): class KeystoneRoleController(service.BaseApplication): - def __init__(self, options): - self.options = options - self.catalog_api = catalog.Manager(options) - self.identity_api = identity.Manager(options) - self.token_api = token.Manager(options) - self.policy_api = policy.Manager(options) + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() super(KeystoneRoleController, self).__init__() def get_user_roles(self, context, user_id, tenant_id=None): @@ -748,12 +740,11 @@ class KeystoneRoleController(service.BaseApplication): class KeystoneServiceController(service.BaseApplication): - def __init__(self, options): - self.options = options - self.catalog_api = catalog.Manager(options) - self.identity_api = identity.Manager(options) - self.token_api = token.Manager(options) - self.policy_api = policy.Manager(options) + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() super(KeystoneServiceController, self).__init__() # CRUD extensions @@ -783,8 +774,7 @@ class KeystoneServiceController(service.BaseApplication): class KeystoneVersionController(service.BaseApplication): - def __init__(self, options): - self.options = options + def __init__(self): super(KeystoneVersionController, self).__init__() def get_version_info(self, context, module='version'): @@ -792,8 +782,7 @@ class KeystoneVersionController(service.BaseApplication): class KeystoneExtensionsController(service.BaseApplication): - def __init__(self, options): - self.options = options + def __init__(self): super(KeystoneExtensionsController, self).__init__() def get_extensions_info(self, context): @@ -803,10 +792,10 @@ class KeystoneExtensionsController(service.BaseApplication): def service_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return KeystoneServiceRouter(conf) + return KeystoneServiceRouter() def admin_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return KeystoneAdminRouter(conf) + return KeystoneAdminRouter() diff --git a/keystonelight/middleware.py b/keystonelight/middleware.py index b8baabc98e..9ab528549d 100644 --- a/keystonelight/middleware.py +++ b/keystonelight/middleware.py @@ -1,8 +1,12 @@ import json +from keystonelight import config from keystonelight import wsgi +CONF = config.CONF + + # Header used to transmit the auth token AUTH_TOKEN_HEADER = 'X-Auth-Token' @@ -34,7 +38,7 @@ class AdminTokenAuthMiddleware(wsgi.Middleware): def process_request(self, request): token = request.headers.get(AUTH_TOKEN_HEADER) context = request.environ.get(CONTEXT_ENV, {}) - context['is_admin'] = (token == self.options['admin_token']) + context['is_admin'] = (token == CONF.admin_token) request.environ[CONTEXT_ENV] = context diff --git a/keystonelight/policy.py b/keystonelight/policy.py index 147c650196..0143878d31 100644 --- a/keystonelight/policy.py +++ b/keystonelight/policy.py @@ -4,14 +4,16 @@ import uuid +from keystonelight import config from keystonelight import utils +CONF = config.CONF + + class Manager(object): - def __init__(self, options): - self.options = options - self.driver = utils.import_object(options['policy_driver'], - options=options) + def __init__(self): + self.driver = utils.import_object(CONF.policy.driver) def can_haz(self, context, target, credentials): """Check whether the given creds can perform action on target.""" diff --git a/keystonelight/service.py b/keystonelight/service.py index 210250cb6d..50e8377c14 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -97,9 +97,8 @@ class BaseApplication(wsgi.Application): class TokenController(BaseApplication): """Validate and pass through calls to TokenManager.""" - def __init__(self, options): - self.token_api = token.Manager(options=options) - self.options = options + def __init__(self): + self.token_api = token.Manager() def validate_token(self, context, token_id): token_info = self.token_api.validate_token(context, token_id) @@ -115,10 +114,9 @@ class IdentityController(BaseApplication): a specific driver. """ - def __init__(self, options): - self.identity_api = identity.Manager(options=options) - self.token_api = token.Manager(options=options) - self.options = options + def __init__(self): + self.identity_api = identity.Manager() + self.token_api = token.Manager() def noop(self, context, *args, **kw): return '' @@ -207,10 +205,9 @@ class IdentityController(BaseApplication): class Router(wsgi.Router): - def __init__(self, options): - self.options = options - self.identity_controller = IdentityController(options) - self.token_controller = TokenController(options) + def __init__(self): + self.identity_controller = IdentityController() + self.token_controller = TokenController() mapper = self._build_map(URLMAP) mapper.connect('/', controller=self.identity_controller, action='noop') @@ -238,6 +235,6 @@ class Router(wsgi.Router): def app_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - return Router(conf) + #conf = global_conf.copy() + #conf.update(local_conf) + return Router() diff --git a/keystonelight/test.py b/keystonelight/test.py index 2045520a02..8af0888048 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -8,6 +8,7 @@ import time from paste import deploy +from keystonelight import config from keystonelight import logging from keystonelight import models from keystonelight import utils @@ -17,7 +18,7 @@ from keystonelight import wsgi ROOTDIR = os.path.dirname(os.path.dirname(__file__)) VENDOR = os.path.join(ROOTDIR, 'vendor') TESTSDIR = os.path.join(ROOTDIR, 'tests') - +CONF = config.CONF cd = os.chdir @@ -94,21 +95,14 @@ class TestCase(unittest.TestCase): for path in self._paths: if path in sys.path: sys.path.remove(path) + CONF.reset() super(TestCase, self).tearDown() - #TODO(termie): probably make this take an argument and use that for `options` def load_backends(self): - """Hacky shortcut to load the backends for data manipulation. - - Expects self.options to have already been set. - - """ - self.identity_api = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_api = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_api = utils.import_object( - self.options['catalog_driver'], options=self.options) + """Hacky shortcut to load the backends for data manipulation.""" + self.identity_api = utils.import_object(CONF.identity.driver) + self.token_api = utils.import_object(CONF.token.driver) + self.catalog_api = utils.import_object(CONF.catalog.driver) def load_fixtures(self, fixtures): """Hacky basic and naive fixture loading based on a python module. @@ -164,44 +158,10 @@ class TestCase(unittest.TestCase): # Service catalog tests need to know the port we ran on. port = server.socket_info['socket'][1] - self._update_server_options(server, 'public_port', port) - self._update_server_options(server, 'admin_port', port) + CONF.public_port = port + CONF.admin_port = port return server - def _update_server_options(self, server, key, value): - """Hack to allow us to make changes to the options used by backends. - - A possible better solution would be to have a global config registry. - - """ - last = server - - applications = [] - - while (hasattr(last, 'applications') - or hasattr(last, 'application') - or hasattr(last, 'options')): - - #logging.debug('UPDATE %s: O %s A %s AS %s', - # last.__class__, - # getattr(last, 'options', None), - # getattr(last, 'application', None), - # getattr(last, 'applications', None)) - if hasattr(last, 'options'): - last.options[key] = value - - # NOTE(termie): paste.urlmap.URLMap stores applications in this format - if hasattr(last, 'applications'): - for app in last.applications: - applications.append(app[1]) - - if hasattr(last, 'application'): - last = last.application - elif len(applications): - last = applications.pop() - else: - break - def client(self, app, *args, **kw): return TestClient(app, *args, **kw) diff --git a/keystonelight/token.py b/keystonelight/token.py index a4c814e572..9e334f185e 100644 --- a/keystonelight/token.py +++ b/keystonelight/token.py @@ -4,15 +4,17 @@ import uuid +from keystonelight import config from keystonelight import logging from keystonelight import utils +CONF = config.CONF + + class Manager(object): - def __init__(self, options): - self.options = options - self.driver = utils.import_object(options['token_driver'], - options=options) + def __init__(self): + self.driver = utils.import_object(CONF.token.driver) def create_token(self, context, data): token = uuid.uuid4().hex diff --git a/keystonelight/wsgi.py b/keystonelight/wsgi.py index 0744fd32a8..401b0f0136 100644 --- a/keystonelight/wsgi.py +++ b/keystonelight/wsgi.py @@ -107,7 +107,7 @@ class Application(object): but using the kwarg passing it shouldn't be necessary. """ - return cls(**local_config) + return cls() def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: @@ -182,12 +182,11 @@ class Middleware(Application): def _factory(app): conf = global_config.copy() conf.update(local_config) - return cls(app, conf) + return cls(app) return _factory - def __init__(self, application, options): + def __init__(self, application): self.application = application - self.options = options def process_request(self, req): """Called on each request. @@ -315,8 +314,7 @@ class ExtensionRouter(Router): Expects to be subclassed. """ - def __init__(self, application, options, mapper): - self.options = options + def __init__(self, application, mapper): self.application = application mapper.connect('{path_info:.*}', controller=self.application) @@ -348,5 +346,5 @@ class ExtensionRouter(Router): def _factory(app): conf = global_config.copy() conf.update(local_config) - return cls(app, conf) + return cls(app) return _factory diff --git a/tests/default.conf b/tests/default.conf index 295fe6ff53..8d54558c69 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -1,32 +1,28 @@ [DEFAULT] -catalog_driver = keystonelight.backends.templated.TemplatedCatalog -identity_driver = keystonelight.backends.kvs.KvsIdentity -token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 35357 admin_token = ADMIN - -# config for TemplatedCatalog, using camelCase because I don't want to do -# translations for keystone compat -catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 -catalog.RegionOne.identity.internalURL = http://localhost:$(admin_port)s/v2.0 -catalog.RegionOne.identity.name = 'Identity Service' - -# fake compute service for now to help novaclient tests work compute_port = 3000 -catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s -catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s -catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s -catalog.RegionOne.compute.name = 'Compute Service' -# for sql backends -sql_connection = sqlite:///bla.db -sql_idle_timeout = 200 -sql_min_pool_size = 5 -sql_max_pool_size = 10 -sql_pool_timeout = 200 +[sql] +connection = sqlite:///bla.db +idle_timeout = 200 +min_pool_size = 5 +max_pool_size = 10 +pool_timeout = 200 + +[identity] +driver = keystonelight.backends.kvs.KvsIdentity + +[catalog] +driver = keystonelight.backends.templated.TemplatedCatalog +template_file = default_catalog.templates + +[token] +driver = keystonelight.backends.kvs.KvsToken + +[policy] +driver = keystonelight.backends.policy.SimpleMatch [filter:debug] diff --git a/tests/default_catalog.templates b/tests/default_catalog.templates new file mode 100644 index 0000000000..c12b5c4ca7 --- /dev/null +++ b/tests/default_catalog.templates @@ -0,0 +1,12 @@ +# config for TemplatedCatalog, using camelCase because I don't want to do +# translations for keystone compat +catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 +catalog.RegionOne.identity.internalURL = http://localhost:$(admin_port)s/v2.0 +catalog.RegionOne.identity.name = 'Identity Service' + +# fake compute service for now to help novaclient tests work +catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.name = 'Compute Service' diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index f50c7706d6..b629a71098 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -1,12 +1,21 @@ [DEFAULT] -catalog_driver = keystonelight.backends.kvs.KvsCatalog -identity_driver = keystonelight.backends.kvs.KvsIdentity -token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 35357 admin_token = ADMIN +[identity] +driver = keystonelight.backends.kvs.KvsIdentity + +[catalog] +driver = keystonelight.backends.kvs.KvsCatalog + +[token] +driver = keystonelight.backends.kvs.KvsToken + +[policy] +driver = keystonelight.backends.policy.SimpleMatch + + [filter:debug] paste.filter_factory = keystonelight.wsgi:Debug.factory diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf index 1930c50cae..1e531112e7 100644 --- a/tests/keystoneclient_compat_master.conf +++ b/tests/keystoneclient_compat_master.conf @@ -1,25 +1,27 @@ [DEFAULT] -catalog_driver = keystonelight.backends.templated.TemplatedCatalog -identity_driver = keystonelight.backends.kvs.KvsIdentity -token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 35357 admin_token = ADMIN -# config for TemplatedCatalog, using camelCase because I don't want to do -# translations for keystone compat -catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 -catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 -catalog.RegionOne.identity.name = 'Identity Service' +[sql] +connection = sqlite:///bla.db +idle_timeout = 200 +min_pool_size = 5 +max_pool_size = 10 +pool_timeout = 200 -# fake compute port for now to help novaclient tests work -compute_port = 3000 -catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v2.0 -catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v2.0 -catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v2.0 -catalog.RegionOne.compute.name = 'Compute Service' +[identity] +driver = keystonelight.backends.kvs.KvsIdentity + +[catalog] +driver = keystonelight.backends.templated.TemplatedCatalog +template_file = default_catalog.templates + +[token] +driver = keystonelight.backends.kvs.KvsToken + +[policy] +driver = keystonelight.backends.policy.SimpleMatch [filter:debug] diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index ca367c476e..05987101a8 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -11,16 +11,14 @@ import default_fixtures class KvsIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(KvsIdentity, self).setUp() - self.options = self.appconfig('default') - self.identity_api = kvs.KvsIdentity(options=self.options, db={}) + self.identity_api = kvs.KvsIdentity(db={}) self.load_fixtures(default_fixtures) class KvsToken(test.TestCase): def setUp(self): super(KvsToken, self).setUp() - options = self.appconfig('default') - self.token_api = kvs.KvsToken(options=options, db={}) + self.token_api = kvs.KvsToken(db={}) def test_token_crud(self): token_id = uuid.uuid4().hex @@ -40,8 +38,7 @@ class KvsToken(test.TestCase): class KvsCatalog(test.TestCase): def setUp(self): super(KvsCatalog, self).setUp() - options = self.appconfig('default') - self.catalog_api = kvs.KvsCatalog(options=options, db={}) + self.catalog_api = kvs.KvsCatalog(db={}) self._load_fixtures() def _load_fixtures(self): diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index bee4ec33fc..2bdc224555 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -1,6 +1,7 @@ import os import uuid +from keystonelight import config from keystonelight import models from keystonelight import test from keystonelight.backends import sql @@ -10,20 +11,27 @@ import test_backend import default_fixtures +CONF = config.CONF + + + class SqlIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(SqlIdentity, self).setUp() - self.options = self.appconfig('default') - os.unlink('bla.db') - migration.db_sync(self.options, 1) - self.identity_api = sql.SqlIdentity(options=self.options) + try: + os.unlink('bla.db') + except Exception: + pass + CONF(config_files=['default.conf']) + migration.db_sync(1) + self.identity_api = sql.SqlIdentity() self.load_fixtures(default_fixtures) #class SqlToken(test_backend_kvs.KvsToken): # def setUp(self): # super(SqlToken, self).setUp() -# self.token_api = sql.SqlToken(options=options) +# self.token_api = sql.SqlToken() # self.load_fixtures(default_fixtures) # def test_token_crud(self): @@ -44,7 +52,7 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests): #class SqlCatalog(test_backend_kvs.KvsCatalog): # def setUp(self): # super(SqlCatalog, self).setUp() -# self.catalog_api = sql.SqlCatalog(options=options) +# self.catalog_api = sql.SqlCatalog() # self._load_fixtures() # def _load_fixtures(self): diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index c9d0ae28ac..b48c5078d3 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -1,16 +1,20 @@ import json from keystonelight import client +from keystonelight import config from keystonelight import models from keystonelight import test import default_fixtures +CONF = config.CONF + + class IdentityApi(test.TestCase): def setUp(self): super(IdentityApi, self).setUp() - self.options = self.appconfig('default') + CONF(config_files=['default.conf']) self.app = self.loadapp('default') self.load_backends() @@ -54,7 +58,7 @@ class IdentityApi(test.TestCase): self.assertDictEquals(self.tenant_bar, data[0]) def test_crud_user(self): - token_id = self.options['admin_token'] + token_id = CONF.admin_token c = client.TestClient(self.app, token=token_id) user_ref = models.User(name='FOO') resp = c.create_user(**user_ref) @@ -84,7 +88,7 @@ class IdentityApi(test.TestCase): #self.assertEquals(delget_resp.status, '404 Not Found') def test_crud_tenant(self): - token_id = self.options['admin_token'] + token_id = CONF.admin_token c = client.TestClient(self.app, token=token_id) tenant_ref = models.Tenant(name='BAZ') resp = c.create_tenant(**tenant_ref) @@ -128,7 +132,7 @@ class IdentityApi(test.TestCase): #self.assertEquals(delget_resp.status, '404 Not Found') def test_crud_extras(self): - token_id = self.options['admin_token'] + token_id = CONF.admin_token user_id = 'foo' tenant_id = 'bar' c = client.TestClient(self.app, token=token_id) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 04714a53ab..933f3c1440 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -1,8 +1,9 @@ +from keystonelight import config from keystonelight import test import default_fixtures - +CONF = config.CONF KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' @@ -12,12 +13,12 @@ class CompatTestCase(test.TestCase): def _public_url(self): public_port = self.public_server.socket_info['socket'][1] - self.options['public_port'] = public_port + CONF.public_port = public_port return "http://localhost:%s/v2.0" % public_port def _admin_url(self): admin_port = self.admin_server.socket_info['socket'][1] - self.options['admin_port'] = admin_port + CONF.admin_port = admin_port return "http://localhost:%s/v2.0" % admin_port def _client(self, **kwargs): @@ -41,7 +42,7 @@ class MasterCompatTestCase(CompatTestCase): from keystoneclient.v2_0 import client as ks_client reload(ks_client) - self.options = self.appconfig('keystoneclient_compat_master') + CONF(config_files=['keystoneclient_compat_master.conf']) self.public_app = self.loadapp('keystoneclient_compat_master', name='main') self.admin_app = self.loadapp('keystoneclient_compat_master', diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index 98c2ffdc5a..9fea88bf22 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -5,12 +5,16 @@ import sys from nose import exc +from keystonelight import config from keystonelight import logging from keystonelight import models from keystonelight import test from keystonelight import utils +CONF = config.CONF + + IDENTITY_API_REPO = 'git://github.com/openstack/identity-api.git' KEYSTONE_REPO = 'git://github.com/openstack/keystone.git' NOVACLIENT_REPO = 'git://github.com/rackspace/python-novaclient.git' @@ -46,16 +50,16 @@ class CompatTestCase(test.TestCase): os.path.join(self.sampledir, 'auth.json'))) # validate_token call - self.tenant_345 = self.identity_backend.create_tenant( + self.tenant_345 = self.identity_api.create_tenant( '345', models.Tenant(id='345', name='My Project')) - self.user_123 = self.identity_backend.create_user( + self.user_123 = self.identity_api.create_user( '123', models.User(id='123', name='jqsmith', tenants=[self.tenant_345['id']], password='password')) - self.extras_123 = self.identity_backend.create_extras( + self.extras_123 = self.identity_api.create_extras( self.user_123['id'], self.tenant_345['id'], dict(roles=[{'id': '234', 'name': 'compute:admin'}, @@ -63,7 +67,7 @@ class CompatTestCase(test.TestCase): 'name': 'object-store:admin', 'tenantId': '1'}], roles_links=[])) - self.token_123 = self.token_backend.create_token( + self.token_123 = self.token_api.create_token( 'ab48a9efdfedb23ty3494', models.Token(id='ab48a9efdfedb23ty3494', expires='2010-11-01T03:32:15-05:00', @@ -79,32 +83,32 @@ class CompatTestCase(test.TestCase): #catalog = json.load(open( # os.path.join(os.path.dirname(__file__), # 'keystone_compat_diablo_sample_catalog.json'))) - #self.catalog_backend.create_catalog(self.user_123['id'], + #self.catalog_api.create_catalog(self.user_123['id'], # self.tenant_345['id'], # catalog) # tenants_for_token call - self.user_foo = self.identity_backend.create_user( + self.user_foo = self.identity_api.create_user( 'foo', models.User(id='foo', name='FOO', tenants=['1234', '3456'])) - self.tenant_1234 = self.identity_backend.create_tenant( + self.tenant_1234 = self.identity_api.create_tenant( '1234', models.Tenant(id='1234', name='ACME Corp', description='A description ...', enabled=True)) - self.tenant_3456 = self.identity_backend.create_tenant( + self.tenant_3456 = self.identity_api.create_tenant( '3456', models.Tenant(id='3456', name='Iron Works', description='A description ...', enabled=True)) - self.token_foo_unscoped = self.token_backend.create_token( + self.token_foo_unscoped = self.token_api.create_token( 'foo_unscoped', models.Token(id='foo_unscoped', user=self.user_foo)) - self.token_foo_scoped = self.token_backend.create_token( + self.token_foo_scoped = self.token_api.create_token( 'foo_scoped', models.Token(id='foo_scoped', user=self.user_foo, @@ -113,18 +117,13 @@ class CompatTestCase(test.TestCase): class DiabloCompatTestCase(CompatTestCase): def setUp(self): + CONF(config_files=['keystone_compat_diablo.conf']) + revdir = test.checkout_vendor(KEYSTONE_REPO, 'stable/diablo') self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) self.app = self.loadapp('keystone_compat_diablo') - self.options = self.appconfig('keystone_compat_diablo') - - self.identity_backend = utils.import_object( - self.options['identity_driver'], options=self.options) - self.token_backend = utils.import_object( - self.options['token_driver'], options=self.options) - self.catalog_backend = utils.import_object( - self.options['catalog_driver'], options=self.options) + self.load_backends() super(DiabloCompatTestCase, self).setUp() def test_authenticate_scoped(self): diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 1a441a4c1d..1e097c5591 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -3,6 +3,7 @@ import json import os import sys +from keystonelight import config from keystonelight import logging from keystonelight import models from keystonelight import test @@ -11,6 +12,7 @@ from keystonelight import utils import default_fixtures +CONF = config.CONF NOVACLIENT_REPO = 'git://github.com/openstack/python-novaclient.git' @@ -30,8 +32,8 @@ class NovaClientCompatMasterTestCase(CompatTestCase): reload(ks_client) reload(base_client) + CONF(config_files=['keystoneclient_compat_master.conf']) self.app = self.loadapp('keystoneclient_compat_master') - self.options = self.appconfig('keystoneclient_compat_master') self.load_backends() self.load_fixtures(default_fixtures) self.server = self.serveapp('keystoneclient_compat_master') @@ -41,7 +43,7 @@ class NovaClientCompatMasterTestCase(CompatTestCase): from novaclient import client as base_client port = self.server.socket_info['socket'][1] - self.options['public_port'] = port + CONF.public_port = port # NOTE(termie): novaclient wants a "/" TypeErrorat the end, keystoneclient does not # NOTE(termie): projectid is apparently sent as tenantName, so... that's From 0f6a9a78d9e8336e5d42b3be7192b426cb1d502b Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 21:18:51 -0800 Subject: [PATCH 134/334] move everything over to the default config --- tests/default.conf | 9 +++- tests/keystoneclient_compat_master.conf | 67 ------------------------- tests/test_keystoneclient_compat.py | 19 +++---- tests/test_novaclient_compat.py | 6 +-- 4 files changed, 21 insertions(+), 80 deletions(-) delete mode 100644 tests/keystoneclient_compat_master.conf diff --git a/tests/default.conf b/tests/default.conf index 8d54558c69..27e3820e2b 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -37,6 +37,9 @@ paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +[filter:crud_extension] +paste.filter_factory = keystonelight.keystone_compat:KeystoneAdminCrudExtension.factory + [app:keystonelight] paste.app_factory = keystonelight.service:app_factory @@ -53,9 +56,13 @@ pipeline = token_auth admin_token_auth json_body debug keystonelight pipeline = token_auth admin_token_auth json_body debug keystone_service [pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug keystone_admin +pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin [composite:main] use = egg:Paste#urlmap / = keystonelight_api /v2.0 = keystone_service_api + +[composite:admin] +use = egg:Paste#urlmap +/v2.0 = keystone_admin_api diff --git a/tests/keystoneclient_compat_master.conf b/tests/keystoneclient_compat_master.conf deleted file mode 100644 index 1e531112e7..0000000000 --- a/tests/keystoneclient_compat_master.conf +++ /dev/null @@ -1,67 +0,0 @@ -[DEFAULT] -public_port = 5000 -admin_port = 35357 -admin_token = ADMIN - -[sql] -connection = sqlite:///bla.db -idle_timeout = 200 -min_pool_size = 5 -max_pool_size = 10 -pool_timeout = 200 - -[identity] -driver = keystonelight.backends.kvs.KvsIdentity - -[catalog] -driver = keystonelight.backends.templated.TemplatedCatalog -template_file = default_catalog.templates - -[token] -driver = keystonelight.backends.kvs.KvsToken - -[policy] -driver = keystonelight.backends.policy.SimpleMatch - - -[filter:debug] -paste.filter_factory = keystonelight.wsgi:Debug.factory - -[filter:token_auth] -paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory - -[filter:admin_token_auth] -paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory - -[filter:json_body] -paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory - -[filter:crud_extension] -paste.filter_factory = keystonelight.keystone_compat:KeystoneAdminCrudExtension.factory - -[app:keystonelight] -paste.app_factory = keystonelight.service:app_factory - -[app:keystone_service] -paste.app_factory = keystonelight.keystone_compat:service_app_factory - -[app:keystone_admin] -paste.app_factory = keystonelight.keystone_compat:admin_app_factory - -[pipeline:keystonelight_api] -pipeline = token_auth admin_token_auth json_body debug keystonelight - -[pipeline:keystone_service_api] -pipeline = token_auth admin_token_auth json_body debug keystone_service - -[pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin - -[composite:main] -use = egg:Paste#urlmap -/ = keystonelight_api -/v2.0 = keystone_service_api - -[composite:admin] -use = egg:Paste#urlmap -/v2.0 = keystone_admin_api diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient_compat.py index 933f3c1440..fc757d6547 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient_compat.py @@ -1,3 +1,4 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 from keystonelight import config from keystonelight import test @@ -42,19 +43,15 @@ class MasterCompatTestCase(CompatTestCase): from keystoneclient.v2_0 import client as ks_client reload(ks_client) - CONF(config_files=['keystoneclient_compat_master.conf']) - self.public_app = self.loadapp('keystoneclient_compat_master', - name='main') - self.admin_app = self.loadapp('keystoneclient_compat_master', - name='admin') + self._config() + self.public_app = self.loadapp('default', name='main') + self.admin_app = self.loadapp('default', name='admin') self.load_backends() self.load_fixtures(default_fixtures) - self.public_server = self.serveapp('keystoneclient_compat_master', - name='main') - self.admin_server = self.serveapp('keystoneclient_compat_master', - name='admin') + self.public_server = self.serveapp('default', name='main') + self.admin_server = self.serveapp('default', name='admin') # TODO(termie): is_admin is being deprecated once the policy stuff # is all working @@ -64,6 +61,10 @@ class MasterCompatTestCase(CompatTestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) + def _config(self): + CONF(config_files=['default.conf']) + + def foo_client(self): return self._client(username='FOO', password='foo2', diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 1e097c5591..387934d529 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -32,11 +32,11 @@ class NovaClientCompatMasterTestCase(CompatTestCase): reload(ks_client) reload(base_client) - CONF(config_files=['keystoneclient_compat_master.conf']) - self.app = self.loadapp('keystoneclient_compat_master') + CONF(config_files=['default.conf']) + self.app = self.loadapp('default') self.load_backends() self.load_fixtures(default_fixtures) - self.server = self.serveapp('keystoneclient_compat_master') + self.server = self.serveapp('default') def test_authenticate_and_tenants(self): from novaclient.keystone import client as ks_client From 4b4ada27cadf9bc5fcaaa9ccab79fdd1400fb450 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 21:31:12 -0800 Subject: [PATCH 135/334] run all teh keystoneclient tests against sql too --- keystonelight/backends/sql/util.py | 16 ++++++++++++++++ tests/backend_sql.conf | 9 +++++++++ tests/test_backend_sql.py | 6 +++--- ...neclient_compat.py => test_keystoneclient.py} | 5 ++--- tests/test_keystoneclient_sql.py | 16 ++++++++++++++++ 5 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 keystonelight/backends/sql/util.py create mode 100644 tests/backend_sql.conf rename tests/{test_keystoneclient_compat.py => test_keystoneclient.py} (98%) create mode 100644 tests/test_keystoneclient_sql.py diff --git a/keystonelight/backends/sql/util.py b/keystonelight/backends/sql/util.py new file mode 100644 index 0000000000..64443ef7dc --- /dev/null +++ b/keystonelight/backends/sql/util.py @@ -0,0 +1,16 @@ +import os + +from keystonelight import config +from keystonelight.backends.sql import migration + + +CONF = config.CONF + + +def setup_test_database(): + # TODO(termie): be smart about this + try: + os.unlink('bla.db') + except Exception: + pass + migration.db_sync() diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf new file mode 100644 index 0000000000..41dccf30e9 --- /dev/null +++ b/tests/backend_sql.conf @@ -0,0 +1,9 @@ +[sql] +connection = sqlite:///bla.db +idle_timeout = 200 +min_pool_size = 5 +max_pool_size = 10 +pool_timeout = 200 + +[identity] +driver = keystonelight.backends.sql.SqlIdentity diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index 2bdc224555..d80e5bfd12 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -5,7 +5,7 @@ from keystonelight import config from keystonelight import models from keystonelight import test from keystonelight.backends import sql -from keystonelight.backends.sql import migration +from keystonelight.backends.sql import util as sql_util import test_backend import default_fixtures @@ -22,8 +22,8 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests): os.unlink('bla.db') except Exception: pass - CONF(config_files=['default.conf']) - migration.db_sync(1) + CONF(config_files=['default.conf', 'backend_sql.conf']) + sql_util.setup_test_database() self.identity_api = sql.SqlIdentity() self.load_fixtures(default_fixtures) diff --git a/tests/test_keystoneclient_compat.py b/tests/test_keystoneclient.py similarity index 98% rename from tests/test_keystoneclient_compat.py rename to tests/test_keystoneclient.py index fc757d6547..0efd3ca889 100644 --- a/tests/test_keystoneclient_compat.py +++ b/tests/test_keystoneclient.py @@ -34,9 +34,9 @@ class CompatTestCase(test.TestCase): return kc -class MasterCompatTestCase(CompatTestCase): +class KcMasterTestCase(CompatTestCase): def setUp(self): - super(MasterCompatTestCase, self).setUp() + super(KcMasterTestCase, self).setUp() revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') self.add_path(revdir) @@ -64,7 +64,6 @@ class MasterCompatTestCase(CompatTestCase): def _config(self): CONF(config_files=['default.conf']) - def foo_client(self): return self._client(username='FOO', password='foo2', diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py new file mode 100644 index 0000000000..eafbd1f76a --- /dev/null +++ b/tests/test_keystoneclient_sql.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +from keystonelight import config +from keystonelight import test +from keystonelight.backends.sql import util as sql_util +from keystonelight.backends.sql import migration + +import test_keystoneclient + + +CONF = config.CONF + + +class KcMasterSqlTestCase(test_keystoneclient.KcMasterTestCase): + def _config(self): + CONF(config_files=['default.conf', 'backend_sql.conf']) + sql_util.setup_test_database() From 13ec79bf48a185a6f359d1e6c013bdfe24b7da53 Mon Sep 17 00:00:00 2001 From: termie Date: Fri, 6 Jan 2012 23:10:18 -0800 Subject: [PATCH 136/334] keystoneclient tests working against sql backend --- keystonelight/backends/kvs.py | 6 +- keystonelight/backends/sql/core.py | 197 ++++++++++++++++++++--------- keystonelight/keystone_compat.py | 27 ++-- keystonelight/service.py | 17 ++- keystonelight/test.py | 12 +- 5 files changed, 180 insertions(+), 79 deletions(-) diff --git a/keystonelight/backends/kvs.py b/keystonelight/backends/kvs.py index 4394f34776..c15e449c80 100644 --- a/keystonelight/backends/kvs.py +++ b/keystonelight/backends/kvs.py @@ -64,10 +64,12 @@ class KvsIdentity(object): return role_ref def list_users(self): - return self.db.get('user_list', []) + user_ids = self.db.get('user_list', []) + return [self.get_user(x) for x in user_ids] def list_roles(self): - return self.db.get('role_list', []) + role_ids = self.db.get('role_list', []) + return [self.get_role(x) for x in role_ids] # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py index 211d9ca261..62528976bd 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystonelight/backends/sql/core.py @@ -63,11 +63,12 @@ class DictBase(object): Includes attributes from joins. """ - local = dict(self) - joined = dict([(k, v) for k, v in self.__dict__.iteritems() - if not k[0] == '_']) - local.update(joined) - return local.iteritems() + return dict([(k, getattr(self, k)) for k in self]) + #local = dict(self) + #joined = dict([(k, v) for k, v in self.__dict__.iteritems() + # if not k[0] == '_']) + #local.update(joined) + #return local.iteritems() # Tables @@ -75,13 +76,51 @@ class User(Base, DictBase): __tablename__ = 'user' id = sql.Column(sql.String(64), primary_key=True) name = sql.Column(sql.String(64), unique=True) - password = sql.Column(sql.String(64)) + #password = sql.Column(sql.String(64)) + extra = sql.Column(JsonBlob()) + + @classmethod + def from_dict(cls, user_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in user_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = user_dict.pop(k) + + user_dict['extra'] = extra + return cls(**user_dict) + + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy class Tenant(Base, DictBase): __tablename__ = 'tenant' id = sql.Column(sql.String(64), primary_key=True) name = sql.Column(sql.String(64), unique=True) + extra = sql.Column(JsonBlob()) + + @classmethod + def from_dict(cls, tenant_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in tenant_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = tenant_dict.pop(k) + + tenant_dict['extra'] = extra + return cls(**tenant_dict) + + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy class Role(Base, DictBase): @@ -143,7 +182,7 @@ class SqlBase(object): engine_args = { "pool_recycle": CONF.sql.idle_timeout, - "echo": False, + "echo": True, } if "sqlite" in connection_dict.drivername: @@ -177,6 +216,7 @@ class SqlIdentity(SqlBase): raise AssertionError('Invalid tenant') tenant_ref = self.get_tenant(tenant_id) + print 'ETESTSET', tenant_ref if tenant_ref: extras_ref = self.get_extras(user_id, tenant_id) else: @@ -186,29 +226,37 @@ class SqlIdentity(SqlBase): def get_tenant(self, tenant_id): session = self.get_session() tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - return tenant_ref + if not tenant_ref: + return + return tenant_ref.to_dict() def get_tenant_by_name(self, tenant_name): session = self.get_session() tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() - return tenant_ref + if not tenant_ref: + return + return tenant_ref.to_dict() def get_user(self, user_id): session = self.get_session() user_ref = session.query(User).filter_by(id=user_id).first() - return user_ref + if not user_ref: + return + return user_ref.to_dict() def get_user_by_name(self, user_name): session = self.get_session() user_ref = session.query(User).filter_by(name=user_name).first() - return user_ref + if not user_ref: + return + return user_ref.to_dict() def get_extras(self, user_id, tenant_id): session = self.get_session() extras_ref = session.query(Extras)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() return getattr(extras_ref, 'data', None) def get_role(self, role_id): @@ -219,7 +267,7 @@ class SqlIdentity(SqlBase): def list_users(self): session = self.get_session() user_refs = session.query(User) - return list(user_refs) + return [x.to_dict() for x in user_refs] def list_roles(self): session = self.get_session() @@ -233,11 +281,13 @@ class SqlIdentity(SqlBase): session.add(UserTenantMembership(user_id=user_id, tenant_id=tenant_id)) def remove_user_from_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.remove(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + session = self.get_session() + membership_ref = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + with session.begin(): + session.delete(membership_ref) def get_tenants_for_user(self, user_id): session = self.get_session() @@ -255,65 +305,84 @@ class SqlIdentity(SqlBase): def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): extras_ref = self.get_extras(user_id, tenant_id) + is_new = False if not extras_ref: + is_new = True extras_ref = {} roles = set(extras_ref.get('roles', [])) roles.add(role_id) extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) + if not is_new: + self.update_extras(user_id, tenant_id, extras_ref) + else: + self.create_extras(user_id, tenant_id, extras_ref) def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): extras_ref = self.get_extras(user_id, tenant_id) + is_new = False if not extras_ref: + is_new = True extras_ref = {} roles = set(extras_ref.get('roles', [])) roles.remove(role_id) extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) + if not is_new: + self.update_extras(user_id, tenant_id, extras_ref) + else: + self.create_extras(user_id, tenant_id, extras_ref) # CRUD def create_user(self, id, user): session = self.get_session() with session.begin(): - session.add(User(**user)) - return user + user_ref = User.from_dict(user) + session.add(user_ref) + return user_ref.to_dict() def update_user(self, id, user): - # get the old name and delete it too - old_user = self.db.get('user-%s' % id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % id, user) - self.db.set('user_name-%s' % user['name'], user) - return user + session = self.get_session() + with session.begin(): + user_ref = session.query(User).filter_by(id=id).first() + old_user_dict = user_ref.to_dict() + for k in user: + old_user_dict[k] = user[k] + new_user = User.from_dict(old_user_dict) + + user_ref.name = new_user.name + user_ref.extra = new_user.extra + return user_ref def delete_user(self, id): - old_user = self.db.get('user-%s' % id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.delete('user-%s' % id) - user_list = set(self.db.get('user_list', [])) - user_list.remove(id) - self.db.set('user_list', list(user_list)) - return None + session = self.get_session() + user_ref = session.query(User).filter_by(id=id).first() + with session.begin(): + session.delete(user_ref) def create_tenant(self, id, tenant): session = self.get_session() with session.begin(): - session.add(Tenant(**tenant)) - return tenant + tenant_ref = Tenant.from_dict(tenant) + session.add(tenant_ref) + return tenant_ref.to_dict() def update_tenant(self, id, tenant): - # get the old name and delete it too - old_tenant = self.db.get('tenant-%s' % id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant + session = self.get_session() + with session.begin(): + tenant_ref = session.query(Tenant).filter_by(id=id).first() + old_tenant_dict = tenant_ref.to_dict() + for k in tenant: + old_tenant_dict[k] = tenant[k] + new_tenant = Tenant.from_dict(old_tenant_dict) + + tenant_ref.name = new_tenant.name + tenant_ref.extra = new_tenant.extra + return tenant_ref def delete_tenant(self, id): - old_tenant = self.db.get('tenant-%s' % id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.delete('tenant-%s' % id) - return None + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=id).first() + with session.begin(): + session.delete(tenant_ref) def create_extras(self, user_id, tenant_id, extras): session = self.get_session() @@ -322,8 +391,17 @@ class SqlIdentity(SqlBase): return extras def update_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras + session = self.get_session() + with session.begin(): + extras_ref = session.query(Extras)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + data = extras_ref.data.copy() + for k in extras: + data[k] = extras[k] + extras_ref.data = data + return extras_ref def delete_extras(self, user_id, tenant_id): self.db.delete('extras-%s-%s' % (tenant_id, user_id)) @@ -336,15 +414,18 @@ class SqlIdentity(SqlBase): return role def update_role(self, id, role): - self.db.set('role-%s' % id, role) - return role + session = self.get_session() + with session.begin(): + role_ref = session.query(Role).filter_by(id=id).first() + for k in role: + role_ref[k] = role[k] + return role_ref def delete_role(self, id): - self.db.delete('role-%s' % id) - role_list = set(self.db.get('role_list', [])) - role_list.remove(id) - self.db.set('role_list', list(role_list)) - return None + session = self.get_session() + role_ref = session.query(Role).filter_by(id=id).first() + with session.begin(): + session.delete(role_ref) class SqlToken(SqlBase): diff --git a/keystonelight/keystone_compat.py b/keystonelight/keystone_compat.py index b720477ca9..c909881d64 100644 --- a/keystonelight/keystone_compat.py +++ b/keystonelight/keystone_compat.py @@ -511,8 +511,10 @@ class KeystoneTenantController(service.BaseApplication): assert token_ref is not None user_ref = token_ref['user'] + tenant_ids = self.identity_api.get_tenants_for_user( + context, user_ref['id']) tenant_refs = [] - for tenant_id in user_ref['tenants']: + for tenant_id in tenant_ids: tenant_refs.append(self.identity_api.get_tenant( context=context, tenant_id=tenant_id)) @@ -537,9 +539,9 @@ class KeystoneTenantController(service.BaseApplication): return {'tenant': tenant} # CRUD Extension - def create_tenant(self, context, **kw): + def create_tenant(self, context, tenant): + tenant_ref = self._normalize_dict(tenant) self.assert_admin(context) - tenant_ref = kw.get('tenant') tenant_id = (tenant_ref.get('id') and tenant_ref.get('id') or uuid.uuid4().hex) @@ -590,23 +592,21 @@ class KeystoneUserController(service.BaseApplication): # NOTE(termie): i can't imagine that this really wants all the data # about every single user in the system... self.assert_admin(context) - user_list = self.identity_api.list_users(context) - return {'users': [{'id': x} for x in user_list]} + user_refs = self.identity_api.list_users(context) + return {'users': user_refs} # CRUD extension def create_user(self, context, user): + user = self._normalize_dict(user) self.assert_admin(context) - tenant_id = user.get('tenantId') - tenants = [] - if tenant_id: - tenants.append(tenant_id) + tenant_id = user.get('tenantId', None) user_id = uuid.uuid4().hex user_ref = user.copy() - #user_ref.pop('tenantId', None) user_ref['id'] = user_id - user_ref['tenants'] = tenants new_user_ref = self.identity_api.create_user( context, user_id, user_ref) + if tenant_id: + self.identity_api.add_user_to_tenant(tenant_id, user_id) return {'user': new_user_ref} # NOTE(termie): this is really more of a patch than a put @@ -656,6 +656,8 @@ class KeystoneRoleController(service.BaseApplication): return {'role': role_ref} def create_role(self, context, role): + role = self._normalize_dict(role) + self.assert_admin(context) role_id = uuid.uuid4().hex role['id'] = role_id role_ref = self.identity_api.create_role(context, role_id, role) @@ -669,8 +671,7 @@ class KeystoneRoleController(service.BaseApplication): self.assert_admin(context) roles = self.identity_api.list_roles(context) # TODO(termie): probably inefficient at some point - return {'roles': [self.identity_api.get_role(context, x) - for x in roles]} + return {'roles': roles} # COMPAT(diablo): CRUD extension def get_role_refs(self, context, user_id): diff --git a/keystonelight/service.py b/keystonelight/service.py index 50e8377c14..0f7dddfefc 100644 --- a/keystonelight/service.py +++ b/keystonelight/service.py @@ -47,6 +47,13 @@ URLMAP = HIGH_LEVEL_CALLS.copy() URLMAP.update(LOW_LEVEL_CALLS) +class SmarterEncoder(json.JSONEncoder): + def default(self, obj): + if not isinstance(obj, dict) and hasattr(obj, 'iteritems'): + return dict(obj.iteritems()) + return super(SmarterEncoder, self).default(obj) + + class BaseApplication(wsgi.Application): @webob.dec.wsgify def __call__(self, req): @@ -76,11 +83,18 @@ class BaseApplication(wsgi.Application): elif isinstance(result, webob.exc.WSGIHTTPException): return result - return json.dumps(result) + return self._serialize(result) + + def _serialize(self, result): + return json.dumps(result, cls=SmarterEncoder) def _normalize_arg(self, arg): return str(arg).replace(':', '_').replace('-', '_') + def _normalize_dict(self, d): + return dict([(self._normalize_arg(k), v) + for (k, v) in d.iteritems()]) + def assert_admin(self, context): if not context['is_admin']: user_token_ref = self.token_api.get_token( @@ -88,6 +102,7 @@ class BaseApplication(wsgi.Application): creds = user_token_ref['extras'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') + print creds # Accept either is_admin or the admin role assert self.policy_api.can_haz(context, ('is_admin:1', 'roles:admin'), diff --git a/keystonelight/test.py b/keystonelight/test.py index 8af0888048..1196b78f57 100644 --- a/keystonelight/test.py +++ b/keystonelight/test.py @@ -8,9 +8,12 @@ import time from paste import deploy +from keystonelight import catalog from keystonelight import config +from keystonelight import identity from keystonelight import logging from keystonelight import models +from keystonelight import token from keystonelight import utils from keystonelight import wsgi @@ -114,20 +117,19 @@ class TestCase(unittest.TestCase): # TODO(termie): doing something from json, probably based on Django's # loaddata will be much preferred. for tenant in fixtures.TENANTS: - rv = self.identity_api.create_tenant( - tenant['id'], models.Tenant(**tenant)) + rv = self.identity_api.create_tenant(tenant['id'], tenant) setattr(self, 'tenant_%s' % tenant['id'], rv) for user in fixtures.USERS: user_copy = user.copy() tenants = user_copy.pop('tenants') - rv = self.identity_api.create_user(user['id'], models.User(**user_copy)) + rv = self.identity_api.create_user(user['id'], user_copy) for tenant_id in tenants: self.identity_api.add_user_to_tenant(tenant_id, user['id']) setattr(self, 'user_%s' % user['id'], rv) for role in fixtures.ROLES: - rv = self.identity_api.create_role(role['id'], models.Role(**role)) + rv = self.identity_api.create_role(role['id'], role) setattr(self, 'role_%s' % role['id'], rv) for extras in fixtures.EXTRAS: @@ -137,7 +139,7 @@ class TestCase(unittest.TestCase): del extras_ref['user_id'] del extras_ref['tenant_id'] rv = self.identity_api.create_extras( - extras['user_id'], extras['tenant_id'], models.Extras(**extras_ref)) + extras['user_id'], extras['tenant_id'], extras_ref) setattr(self, 'extras_%s%s' % (extras['user_id'], extras['tenant_id']), rv) From 763013c5264e87dc8075d50117e5e394fccef1a7 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 9 Jan 2012 10:28:55 -0800 Subject: [PATCH 137/334] renaming keystonelight to keystone --- .gitignore | 1 + bin/keystone | 6 ++-- bin/{ksl => ks} | 2 +- {keystonelight => keystone}/__init__.py | 0 .../backends/__init__.py | 0 {keystonelight => keystone}/backends/kvs.py | 0 {keystonelight => keystone}/backends/pam.py | 0 .../backends/policy.py | 0 keystone/backends/sql/__init__.py | 1 + .../backends/sql/core.py | 4 +-- .../backends/sql/migrate_repo/README | 0 .../backends/sql/migrate_repo/__init__.py | 0 .../backends/sql/migrate_repo/manage.py | 0 .../backends/sql/migrate_repo/migrate.cfg | 0 .../versions/001_add_initial_tables.py | 2 +- .../sql/migrate_repo/versions/__init__.py | 0 .../backends/sql/migration.py | 2 +- .../backends/sql/util.py | 4 +-- .../backends/templated.py | 6 ++-- {keystonelight => keystone}/catalog.py | 4 +-- {keystonelight => keystone}/cfg.py | 0 {keystonelight => keystone}/client.py | 4 +-- {keystonelight => keystone}/config.py | 2 +- {keystonelight => keystone}/identity.py | 4 +-- .../keystone_compat.py | 12 +++---- {keystonelight => keystone}/logging.py | 0 {keystonelight => keystone}/middleware.py | 4 +-- {keystonelight => keystone}/models.py | 0 {keystonelight => keystone}/policy.py | 6 ++-- {keystonelight => keystone}/service.py | 7 ++-- {keystonelight => keystone}/test.py | 18 +++++------ {keystonelight => keystone}/token.py | 6 ++-- {keystonelight => keystone}/utils.py | 2 +- {keystonelight => keystone}/wsgi.py | 0 keystonelight/backends/sql/__init__.py | 1 - run_tests.sh | 2 +- setup.py | 4 +-- tests/backend_sql.conf | 2 +- tests/default.conf | 32 +++++++++---------- tests/keystone_compat_diablo.conf | 30 ++++++++--------- tests/test_backend_kvs.py | 6 ++-- tests/test_backend_sql.py | 10 +++--- tests/test_identity_api.py | 8 ++--- tests/test_keystoneclient.py | 4 +-- tests/test_keystoneclient_sql.py | 8 ++--- tests/test_legacy_compat.py | 10 +++--- tests/test_novaclient_compat.py | 10 +++--- 47 files changed, 110 insertions(+), 114 deletions(-) rename bin/{ksl => ks} (98%) rename {keystonelight => keystone}/__init__.py (100%) rename {keystonelight => keystone}/backends/__init__.py (100%) rename {keystonelight => keystone}/backends/kvs.py (100%) rename {keystonelight => keystone}/backends/pam.py (100%) rename {keystonelight => keystone}/backends/policy.py (100%) create mode 100644 keystone/backends/sql/__init__.py rename {keystonelight => keystone}/backends/sql/core.py (99%) rename {keystonelight => keystone}/backends/sql/migrate_repo/README (100%) rename {keystonelight => keystone}/backends/sql/migrate_repo/__init__.py (100%) rename {keystonelight => keystone}/backends/sql/migrate_repo/manage.py (100%) rename {keystonelight => keystone}/backends/sql/migrate_repo/migrate.cfg (100%) rename {keystonelight => keystone}/backends/sql/migrate_repo/versions/001_add_initial_tables.py (89%) rename {keystonelight => keystone}/backends/sql/migrate_repo/versions/__init__.py (100%) rename {keystonelight => keystone}/backends/sql/migration.py (98%) rename {keystonelight => keystone}/backends/sql/util.py (68%) rename {keystonelight => keystone}/backends/templated.py (95%) rename {keystonelight => keystone}/catalog.py (92%) rename {keystonelight => keystone}/cfg.py (100%) rename {keystonelight => keystone}/client.py (97%) rename {keystonelight => keystone}/config.py (97%) rename {keystonelight => keystone}/identity.py (98%) rename {keystonelight => keystone}/keystone_compat.py (99%) rename {keystonelight => keystone}/logging.py (100%) rename {keystonelight => keystone}/middleware.py (97%) rename {keystonelight => keystone}/models.py (100%) rename {keystonelight => keystone}/policy.py (83%) rename {keystonelight => keystone}/service.py (98%) rename {keystonelight => keystone}/test.py (95%) rename {keystonelight => keystone}/token.py (86%) rename {keystonelight => keystone}/utils.py (98%) rename {keystonelight => keystone}/wsgi.py (100%) delete mode 100644 keystonelight/backends/sql/__init__.py diff --git a/.gitignore b/.gitignore index bf562ab7f1..c20afefb53 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ run_tests.log covhtml pep8.txt nosetests.xml +bla.db diff --git a/bin/keystone b/bin/keystone index 7775b20337..58e7b3a941 100755 --- a/bin/keystone +++ b/bin/keystone @@ -12,13 +12,13 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) if os.path.exists(os.path.join(possible_topdir, - 'keystonelight', + 'keystone', '__init__.py')): sys.path.insert(0, possible_topdir) from paste import deploy -from keystonelight import wsgi +from keystone import wsgi def create_server(name, port): @@ -49,6 +49,6 @@ if __name__ == '__main__': servers = [] servers.append(create_server('keystone_admin', int(options['admin_port']))) - servers.append(create_server('keystonelight', + servers.append(create_server('keystone', int(options['public_port']))) serve(*servers) diff --git a/bin/ksl b/bin/ks similarity index 98% rename from bin/ksl rename to bin/ks index 0a9d7736f7..98a6ded243 100755 --- a/bin/ksl +++ b/bin/ks @@ -5,7 +5,7 @@ import sys import cli.app import cli.log -from keystonelight import client +from keystone import client DEFAULT_PARAMS = ( diff --git a/keystonelight/__init__.py b/keystone/__init__.py similarity index 100% rename from keystonelight/__init__.py rename to keystone/__init__.py diff --git a/keystonelight/backends/__init__.py b/keystone/backends/__init__.py similarity index 100% rename from keystonelight/backends/__init__.py rename to keystone/backends/__init__.py diff --git a/keystonelight/backends/kvs.py b/keystone/backends/kvs.py similarity index 100% rename from keystonelight/backends/kvs.py rename to keystone/backends/kvs.py diff --git a/keystonelight/backends/pam.py b/keystone/backends/pam.py similarity index 100% rename from keystonelight/backends/pam.py rename to keystone/backends/pam.py diff --git a/keystonelight/backends/policy.py b/keystone/backends/policy.py similarity index 100% rename from keystonelight/backends/policy.py rename to keystone/backends/policy.py diff --git a/keystone/backends/sql/__init__.py b/keystone/backends/sql/__init__.py new file mode 100644 index 0000000000..c52d6cbce2 --- /dev/null +++ b/keystone/backends/sql/__init__.py @@ -0,0 +1 @@ +from keystone.backends.sql.core import * diff --git a/keystonelight/backends/sql/core.py b/keystone/backends/sql/core.py similarity index 99% rename from keystonelight/backends/sql/core.py rename to keystone/backends/sql/core.py index 62528976bd..7f56bdc287 100644 --- a/keystonelight/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -10,8 +10,8 @@ import sqlalchemy.orm import sqlalchemy.pool import sqlalchemy.engine.url -from keystonelight import config -from keystonelight import models +from keystone import config +from keystone import models CONF = config.CONF diff --git a/keystonelight/backends/sql/migrate_repo/README b/keystone/backends/sql/migrate_repo/README similarity index 100% rename from keystonelight/backends/sql/migrate_repo/README rename to keystone/backends/sql/migrate_repo/README diff --git a/keystonelight/backends/sql/migrate_repo/__init__.py b/keystone/backends/sql/migrate_repo/__init__.py similarity index 100% rename from keystonelight/backends/sql/migrate_repo/__init__.py rename to keystone/backends/sql/migrate_repo/__init__.py diff --git a/keystonelight/backends/sql/migrate_repo/manage.py b/keystone/backends/sql/migrate_repo/manage.py similarity index 100% rename from keystonelight/backends/sql/migrate_repo/manage.py rename to keystone/backends/sql/migrate_repo/manage.py diff --git a/keystonelight/backends/sql/migrate_repo/migrate.cfg b/keystone/backends/sql/migrate_repo/migrate.cfg similarity index 100% rename from keystonelight/backends/sql/migrate_repo/migrate.cfg rename to keystone/backends/sql/migrate_repo/migrate.cfg diff --git a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py b/keystone/backends/sql/migrate_repo/versions/001_add_initial_tables.py similarity index 89% rename from keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py rename to keystone/backends/sql/migrate_repo/versions/001_add_initial_tables.py index ecfd6f507d..92c9e6cd1e 100644 --- a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py +++ b/keystone/backends/sql/migrate_repo/versions/001_add_initial_tables.py @@ -1,7 +1,7 @@ from sqlalchemy import * from migrate import * -from keystonelight.backends import sql +from keystone.backends import sql def upgrade(migrate_engine): diff --git a/keystonelight/backends/sql/migrate_repo/versions/__init__.py b/keystone/backends/sql/migrate_repo/versions/__init__.py similarity index 100% rename from keystonelight/backends/sql/migrate_repo/versions/__init__.py rename to keystone/backends/sql/migrate_repo/versions/__init__.py diff --git a/keystonelight/backends/sql/migration.py b/keystone/backends/sql/migration.py similarity index 98% rename from keystonelight/backends/sql/migration.py rename to keystone/backends/sql/migration.py index 106bcea7ee..11d459a1ae 100644 --- a/keystonelight/backends/sql/migration.py +++ b/keystone/backends/sql/migration.py @@ -22,7 +22,7 @@ import sys import sqlalchemy from migrate.versioning import api as versioning_api -from keystonelight import config +from keystone import config CONF = config.CONF diff --git a/keystonelight/backends/sql/util.py b/keystone/backends/sql/util.py similarity index 68% rename from keystonelight/backends/sql/util.py rename to keystone/backends/sql/util.py index 64443ef7dc..498ee6c2a0 100644 --- a/keystonelight/backends/sql/util.py +++ b/keystone/backends/sql/util.py @@ -1,7 +1,7 @@ import os -from keystonelight import config -from keystonelight.backends.sql import migration +from keystone import config +from keystone.backends.sql import migration CONF = config.CONF diff --git a/keystonelight/backends/templated.py b/keystone/backends/templated.py similarity index 95% rename from keystonelight/backends/templated.py rename to keystone/backends/templated.py index 155839b2be..8f00fc41bd 100644 --- a/keystonelight/backends/templated.py +++ b/keystone/backends/templated.py @@ -1,6 +1,6 @@ -from keystonelight import config -from keystonelight import logging -from keystonelight.backends import kvs +from keystone import config +from keystone import logging +from keystone.backends import kvs CONF = config.CONF diff --git a/keystonelight/catalog.py b/keystone/catalog.py similarity index 92% rename from keystonelight/catalog.py rename to keystone/catalog.py index b4d472732e..96cfd4586e 100644 --- a/keystonelight/catalog.py +++ b/keystone/catalog.py @@ -2,8 +2,8 @@ # the catalog interfaces -from keystonelight import config -from keystonelight import utils +from keystone import config +from keystone import utils CONF = config.CONF diff --git a/keystonelight/cfg.py b/keystone/cfg.py similarity index 100% rename from keystonelight/cfg.py rename to keystone/cfg.py diff --git a/keystonelight/client.py b/keystone/client.py similarity index 97% rename from keystonelight/client.py rename to keystone/client.py index f5a78c5bbd..899e8d168b 100644 --- a/keystonelight/client.py +++ b/keystone/client.py @@ -6,8 +6,8 @@ import json import httplib2 import webob -from keystonelight import service -from keystonelight import wsgi +from keystone import service +from keystone import wsgi URLMAP = service.URLMAP diff --git a/keystonelight/config.py b/keystone/config.py similarity index 97% rename from keystonelight/config.py rename to keystone/config.py index 03d6a24b81..7ce9f3ca49 100644 --- a/keystonelight/config.py +++ b/keystone/config.py @@ -1,6 +1,6 @@ -from keystonelight import cfg +from keystone import cfg class Config(cfg.ConfigOpts): diff --git a/keystonelight/identity.py b/keystone/identity.py similarity index 98% rename from keystonelight/identity.py rename to keystone/identity.py index 93e8d309a5..6e606949df 100644 --- a/keystonelight/identity.py +++ b/keystone/identity.py @@ -2,8 +2,8 @@ # backends will make use of them to return something that conforms to their # apis -from keystonelight import config -from keystonelight import utils +from keystone import config +from keystone import utils CONF = config.CONF diff --git a/keystonelight/keystone_compat.py b/keystone/keystone_compat.py similarity index 99% rename from keystonelight/keystone_compat.py rename to keystone/keystone_compat.py index c909881d64..f87c8d46ac 100644 --- a/keystonelight/keystone_compat.py +++ b/keystone/keystone_compat.py @@ -9,12 +9,12 @@ import uuid import routes from webob import exc -from keystonelight import catalog -from keystonelight import identity -from keystonelight import policy -from keystonelight import service -from keystonelight import token -from keystonelight import wsgi +from keystone import catalog +from keystone import identity +from keystone import policy +from keystone import service +from keystone import token +from keystone import wsgi class KeystoneAdminRouter(wsgi.Router): diff --git a/keystonelight/logging.py b/keystone/logging.py similarity index 100% rename from keystonelight/logging.py rename to keystone/logging.py diff --git a/keystonelight/middleware.py b/keystone/middleware.py similarity index 97% rename from keystonelight/middleware.py rename to keystone/middleware.py index 9ab528549d..5bac8fb27b 100644 --- a/keystonelight/middleware.py +++ b/keystone/middleware.py @@ -1,7 +1,7 @@ import json -from keystonelight import config -from keystonelight import wsgi +from keystone import config +from keystone import wsgi CONF = config.CONF diff --git a/keystonelight/models.py b/keystone/models.py similarity index 100% rename from keystonelight/models.py rename to keystone/models.py diff --git a/keystonelight/policy.py b/keystone/policy.py similarity index 83% rename from keystonelight/policy.py rename to keystone/policy.py index 0143878d31..f5c3c6d4c5 100644 --- a/keystonelight/policy.py +++ b/keystone/policy.py @@ -2,10 +2,8 @@ # the catalog interfaces -import uuid - -from keystonelight import config -from keystonelight import utils +from keystone import config +from keystone import utils CONF = config.CONF diff --git a/keystonelight/service.py b/keystone/service.py similarity index 98% rename from keystonelight/service.py rename to keystone/service.py index 0f7dddfefc..fa77830496 100644 --- a/keystonelight/service.py +++ b/keystone/service.py @@ -6,10 +6,9 @@ import routes import webob.dec import webob.exc -from keystonelight import identity -from keystonelight import token -from keystonelight import utils -from keystonelight import wsgi +from keystone import identity +from keystone import token +from keystone import wsgi HIGH_LEVEL_CALLS = { diff --git a/keystonelight/test.py b/keystone/test.py similarity index 95% rename from keystonelight/test.py rename to keystone/test.py index 1196b78f57..bfd6a0ef4f 100644 --- a/keystonelight/test.py +++ b/keystone/test.py @@ -1,5 +1,3 @@ -import ConfigParser -import logging import os import unittest import subprocess @@ -8,14 +6,14 @@ import time from paste import deploy -from keystonelight import catalog -from keystonelight import config -from keystonelight import identity -from keystonelight import logging -from keystonelight import models -from keystonelight import token -from keystonelight import utils -from keystonelight import wsgi +from keystone import catalog +from keystone import config +from keystone import identity +from keystone import logging +from keystone import models +from keystone import token +from keystone import utils +from keystone import wsgi ROOTDIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/keystonelight/token.py b/keystone/token.py similarity index 86% rename from keystonelight/token.py rename to keystone/token.py index 9e334f185e..194767bc8a 100644 --- a/keystonelight/token.py +++ b/keystone/token.py @@ -4,9 +4,9 @@ import uuid -from keystonelight import config -from keystonelight import logging -from keystonelight import utils +from keystone import config +from keystone import logging +from keystone import utils CONF = config.CONF diff --git a/keystonelight/utils.py b/keystone/utils.py similarity index 98% rename from keystonelight/utils.py rename to keystone/utils.py index ea4404f3c4..139c563122 100644 --- a/keystonelight/utils.py +++ b/keystone/utils.py @@ -20,7 +20,7 @@ import subprocess import sys -from keystonelight import logging +from keystone import logging def import_class(import_str): diff --git a/keystonelight/wsgi.py b/keystone/wsgi.py similarity index 100% rename from keystonelight/wsgi.py rename to keystone/wsgi.py diff --git a/keystonelight/backends/sql/__init__.py b/keystonelight/backends/sql/__init__.py deleted file mode 100644 index 38ecafd5d9..0000000000 --- a/keystonelight/backends/sql/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from keystonelight.backends.sql.core import * diff --git a/run_tests.sh b/run_tests.sh index d561a7b925..be245a9afe 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -86,7 +86,7 @@ function run_pep8 { ignore_dirs="*ajaxterm*" GLOBIGNORE="$ignore_scripts:$ignore_files:$ignore_dirs" srcfiles=`find bin -type f ! -name .*.swp` - srcfiles+=" keystonelight" + srcfiles+=" keystone" # Just run PEP8 in current environment ${wrapper} pep8 --repeat --show-pep8 --show-source \ --ignore=E202,E111 \ diff --git a/setup.py b/setup.py index 21099e48e1..bf7a7833a6 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,13 @@ from setuptools import setup, find_packages -setup(name='keystonelight', +setup(name='keystone', version='1.0', description="Authentication service for OpenStack", author='OpenStack, LLC.', author_email='openstack@lists.launchpad.net', url='http://www.openstack.org', packages=find_packages(exclude=['test', 'bin']), - scripts=['bin/keystone', 'bin/ksl'], + scripts=['bin/keystone', 'bin/ks'], zip_safe=False, install_requires=['setuptools'], ) diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf index 41dccf30e9..ca527e9af2 100644 --- a/tests/backend_sql.conf +++ b/tests/backend_sql.conf @@ -6,4 +6,4 @@ max_pool_size = 10 pool_timeout = 200 [identity] -driver = keystonelight.backends.sql.SqlIdentity +driver = keystone.backends.sql.SqlIdentity diff --git a/tests/default.conf b/tests/default.conf index 27e3820e2b..3062f4b317 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -12,45 +12,45 @@ max_pool_size = 10 pool_timeout = 200 [identity] -driver = keystonelight.backends.kvs.KvsIdentity +driver = keystone.backends.kvs.KvsIdentity [catalog] -driver = keystonelight.backends.templated.TemplatedCatalog +driver = keystone.backends.templated.TemplatedCatalog template_file = default_catalog.templates [token] -driver = keystonelight.backends.kvs.KvsToken +driver = keystone.backends.kvs.KvsToken [policy] -driver = keystonelight.backends.policy.SimpleMatch +driver = keystone.backends.policy.SimpleMatch [filter:debug] -paste.filter_factory = keystonelight.wsgi:Debug.factory +paste.filter_factory = keystone.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] -paste.filter_factory = keystonelight.keystone_compat:KeystoneAdminCrudExtension.factory +paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory -[app:keystonelight] -paste.app_factory = keystonelight.service:app_factory +[app:keystone] +paste.app_factory = keystone.service:app_factory [app:keystone_service] -paste.app_factory = keystonelight.keystone_compat:service_app_factory +paste.app_factory = keystone.keystone_compat:service_app_factory [app:keystone_admin] -paste.app_factory = keystonelight.keystone_compat:admin_app_factory +paste.app_factory = keystone.keystone_compat:admin_app_factory -[pipeline:keystonelight_api] -pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_api] +pipeline = token_auth admin_token_auth json_body debug keystone [pipeline:keystone_service_api] pipeline = token_auth admin_token_auth json_body debug keystone_service @@ -60,7 +60,7 @@ pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_a [composite:main] use = egg:Paste#urlmap -/ = keystonelight_api +/ = keystone_api /v2.0 = keystone_service_api [composite:admin] diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index b629a71098..842bb810dc 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -4,41 +4,41 @@ admin_port = 35357 admin_token = ADMIN [identity] -driver = keystonelight.backends.kvs.KvsIdentity +driver = keystone.backends.kvs.KvsIdentity [catalog] -driver = keystonelight.backends.kvs.KvsCatalog +driver = keystone.backends.kvs.KvsCatalog [token] -driver = keystonelight.backends.kvs.KvsToken +driver = keystone.backends.kvs.KvsToken [policy] -driver = keystonelight.backends.policy.SimpleMatch +driver = keystone.backends.policy.SimpleMatch [filter:debug] -paste.filter_factory = keystonelight.wsgi:Debug.factory +paste.filter_factory = keystone.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory -[app:keystonelight] -paste.app_factory = keystonelight.service:app_factory +[app:keystone] +paste.app_factory = keystone.service:app_factory [app:keystone_service] -paste.app_factory = keystonelight.keystone_compat:service_app_factory +paste.app_factory = keystone.keystone_compat:service_app_factory [app:keystone_admin] -paste.app_factory = keystonelight.keystone_compat:admin_app_factory +paste.app_factory = keystone.keystone_compat:admin_app_factory -[pipeline:keystonelight_api] -pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_api] +pipeline = token_auth admin_token_auth json_body debug keystone [pipeline:keystone_service_api] pipeline = token_auth admin_token_auth json_body debug keystone_service @@ -48,5 +48,5 @@ pipeline = token_auth admin_token_auth json_body debug keystone_admin [composite:main] use = egg:Paste#urlmap -/ = keystonelight_api +/ = keystone_api /v2.0 = keystone_service_api diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 05987101a8..a5a67ff2d3 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -1,8 +1,8 @@ import uuid -from keystonelight import models -from keystonelight import test -from keystonelight.backends import kvs +from keystone import models +from keystone import test +from keystone.backends import kvs import test_backend import default_fixtures diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index d80e5bfd12..3dac570484 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -1,11 +1,11 @@ import os import uuid -from keystonelight import config -from keystonelight import models -from keystonelight import test -from keystonelight.backends import sql -from keystonelight.backends.sql import util as sql_util +from keystone import config +from keystone import models +from keystone import test +from keystone.backends import sql +from keystone.backends.sql import util as sql_util import test_backend import default_fixtures diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index b48c5078d3..ef13ab1440 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -1,9 +1,9 @@ import json -from keystonelight import client -from keystonelight import config -from keystonelight import models -from keystonelight import test +from keystone import client +from keystone import config +from keystone import models +from keystone import test import default_fixtures diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 0efd3ca889..3da7a77f40 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -from keystonelight import config -from keystonelight import test +from keystone import config +from keystone import test import default_fixtures diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py index eafbd1f76a..b19aff8fb4 100644 --- a/tests/test_keystoneclient_sql.py +++ b/tests/test_keystoneclient_sql.py @@ -1,8 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -from keystonelight import config -from keystonelight import test -from keystonelight.backends.sql import util as sql_util -from keystonelight.backends.sql import migration +from keystone import config +from keystone import test +from keystone.backends.sql import util as sql_util +from keystone.backends.sql import migration import test_keystoneclient diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index 9fea88bf22..1525acaabd 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -5,11 +5,11 @@ import sys from nose import exc -from keystonelight import config -from keystonelight import logging -from keystonelight import models -from keystonelight import test -from keystonelight import utils +from keystone import config +from keystone import logging +from keystone import models +from keystone import test +from keystone import utils CONF = config.CONF diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 387934d529..d762c5f1e2 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -3,11 +3,11 @@ import json import os import sys -from keystonelight import config -from keystonelight import logging -from keystonelight import models -from keystonelight import test -from keystonelight import utils +from keystone import config +from keystone import logging +from keystone import models +from keystone import test +from keystone import utils import default_fixtures From 19675450d4ac089b4a978ca62185d1e929094ad8 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 13:11:11 -0800 Subject: [PATCH 138/334] remove references to keystone light --- bin/ks | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ks b/bin/ks index 98a6ded243..4eaceeca3a 100755 --- a/bin/ks +++ b/bin/ks @@ -50,7 +50,7 @@ class LoadData(BaseApp): self.add_param('fixture', nargs='+') def main(self): - """Given some fixtures, create the appropriate data in Keystone Light.""" + """Given some fixtures, create the appropriate data in Keystone.""" pass @@ -64,7 +64,7 @@ class CrudCommands(BaseApp): self.add_param('keyvalues', nargs='+') def main(self): - """Given some keyvalues create the appropriate data in Keystone Light.""" + """Given some keyvalues create the appropriate data in Keystone.""" c = client.HttpClient(self.params.url, token=self.params.token) action_name = self.ACTION_MAP[self.params.action] kv = self._parse_keyvalues(self.params.keyvalues) @@ -97,7 +97,7 @@ class Auth(BaseApp): self.add_param('keyvalues', nargs='+') def main(self): - """Attempt to authenticate against the Keystone Light API.""" + """Attempt to authenticate against the Keystone API.""" c = client.HttpClient(self.params.url, token=self.params.token) kv = self._parse_keyvalues(self.params.keyvalues) resp = c.authenticate(**kv) From a84930a8265b8e917e75fe4814be706721596253 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 13:25:20 -0800 Subject: [PATCH 139/334] add basic sphinx doc bits --- docs/Makefile | 158 +++++++++++ docs/source/conf.py | 247 ++++++++++++++++++ docs/source/index.rst | 22 ++ docs/source/keystone.backends.rst | 42 +++ .../keystone.backends.sql.migrate_repo.rst | 18 ++ ...one.backends.sql.migrate_repo.versions.rst | 11 + docs/source/keystone.backends.sql.rst | 42 +++ docs/source/keystone.rst | 130 +++++++++ docs/source/modules.rst | 9 + 9 files changed, 679 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/keystone.backends.rst create mode 100644 docs/source/keystone.backends.sql.migrate_repo.rst create mode 100644 docs/source/keystone.backends.sql.migrate_repo.versions.rst create mode 100644 docs/source/keystone.backends.sql.rst create mode 100644 docs/source/keystone.rst create mode 100644 docs/source/modules.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000..5083ea0d1b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,158 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build +SOURCEDIR = source +SPHINXAPIDOC = sphinx-apidoc + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +apidoc: + $(SPHINXAPIDOC) -o $(SOURCEDIR) ../keystone + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/keystone.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/keystone.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/keystone" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/keystone" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000..1f7785ef0a --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- +# +# keystone documentation build configuration file, created by +# sphinx-quickstart on Mon Jan 9 12:02:59 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'keystone' +copyright = u'2012, termie' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '2012.1' +# The full version, including alpha/beta/rc tags. +release = '2012.1-dev' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'keystonedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'keystone.tex', u'keystone Documentation', + u'termie', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'keystone', u'keystone Documentation', + [u'termie'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'keystone', u'keystone Documentation', + u'termie', 'keystone', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000000..cd7c96a8ff --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +.. keystone documentation master file, created by + sphinx-quickstart on Mon Jan 9 12:02:59 2012. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to keystone's documentation! +==================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/source/keystone.backends.rst b/docs/source/keystone.backends.rst new file mode 100644 index 0000000000..8641ad9e16 --- /dev/null +++ b/docs/source/keystone.backends.rst @@ -0,0 +1,42 @@ +backends Package +================ + +:mod:`kvs` Module +----------------- + +.. automodule:: keystone.backends.kvs + :members: + :undoc-members: + :show-inheritance: + +:mod:`pam` Module +----------------- + +.. automodule:: keystone.backends.pam + :members: + :undoc-members: + :show-inheritance: + +:mod:`policy` Module +-------------------- + +.. automodule:: keystone.backends.policy + :members: + :undoc-members: + :show-inheritance: + +:mod:`templated` Module +----------------------- + +.. automodule:: keystone.backends.templated + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + keystone.backends.sql + diff --git a/docs/source/keystone.backends.sql.migrate_repo.rst b/docs/source/keystone.backends.sql.migrate_repo.rst new file mode 100644 index 0000000000..d47a101eaa --- /dev/null +++ b/docs/source/keystone.backends.sql.migrate_repo.rst @@ -0,0 +1,18 @@ +migrate_repo Package +==================== + +:mod:`manage` Module +-------------------- + +.. automodule:: keystone.backends.sql.migrate_repo.manage + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + keystone.backends.sql.migrate_repo.versions + diff --git a/docs/source/keystone.backends.sql.migrate_repo.versions.rst b/docs/source/keystone.backends.sql.migrate_repo.versions.rst new file mode 100644 index 0000000000..85e1ce8629 --- /dev/null +++ b/docs/source/keystone.backends.sql.migrate_repo.versions.rst @@ -0,0 +1,11 @@ +versions Package +================ + +:mod:`001_add_initial_tables` Module +------------------------------------ + +.. automodule:: keystone.backends.sql.migrate_repo.versions.001_add_initial_tables + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/keystone.backends.sql.rst b/docs/source/keystone.backends.sql.rst new file mode 100644 index 0000000000..91c9f735dd --- /dev/null +++ b/docs/source/keystone.backends.sql.rst @@ -0,0 +1,42 @@ +sql Package +=========== + +:mod:`sql` Package +------------------ + +.. automodule:: keystone.backends.sql + :members: + :undoc-members: + :show-inheritance: + +:mod:`core` Module +------------------ + +.. automodule:: keystone.backends.sql.core + :members: + :undoc-members: + :show-inheritance: + +:mod:`migration` Module +----------------------- + +.. automodule:: keystone.backends.sql.migration + :members: + :undoc-members: + :show-inheritance: + +:mod:`util` Module +------------------ + +.. automodule:: keystone.backends.sql.util + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + keystone.backends.sql.migrate_repo + diff --git a/docs/source/keystone.rst b/docs/source/keystone.rst new file mode 100644 index 0000000000..d4737f0672 --- /dev/null +++ b/docs/source/keystone.rst @@ -0,0 +1,130 @@ +keystone Package +================ + +:mod:`catalog` Module +--------------------- + +.. automodule:: keystone.catalog + :members: + :undoc-members: + :show-inheritance: + +:mod:`cfg` Module +----------------- + +.. automodule:: keystone.cfg + :members: + :undoc-members: + :show-inheritance: + +:mod:`client` Module +-------------------- + +.. automodule:: keystone.client + :members: + :undoc-members: + :show-inheritance: + +:mod:`config` Module +-------------------- + +.. automodule:: keystone.config + :members: + :undoc-members: + :show-inheritance: + +:mod:`identity` Module +---------------------- + +.. automodule:: keystone.identity + :members: + :undoc-members: + :show-inheritance: + +:mod:`keystone_compat` Module +----------------------------- + +.. automodule:: keystone.keystone_compat + :members: + :undoc-members: + :show-inheritance: + +:mod:`logging` Module +--------------------- + +.. automodule:: keystone.logging + :members: + :undoc-members: + :show-inheritance: + +:mod:`middleware` Module +------------------------ + +.. automodule:: keystone.middleware + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +-------------------- + +.. automodule:: keystone.models + :members: + :undoc-members: + :show-inheritance: + +:mod:`policy` Module +-------------------- + +.. automodule:: keystone.policy + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +--------------------- + +.. automodule:: keystone.service + :members: + :undoc-members: + :show-inheritance: + +:mod:`test` Module +------------------ + +.. automodule:: keystone.test + :members: + :undoc-members: + :show-inheritance: + +:mod:`token` Module +------------------- + +.. automodule:: keystone.token + :members: + :undoc-members: + :show-inheritance: + +:mod:`utils` Module +------------------- + +.. automodule:: keystone.utils + :members: + :undoc-members: + :show-inheritance: + +:mod:`wsgi` Module +------------------ + +.. automodule:: keystone.wsgi + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + keystone.backends + diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000000..e071e10801 --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,9 @@ +.. +== + +.. toctree:: + :maxdepth: 4 + + keystone + run_tests + setup From 836244267812cde323b583fb554c731e48bd9aaa Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 13:27:17 -0800 Subject: [PATCH 140/334] version number in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf7a7833a6..ef033b0fda 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='keystone', - version='1.0', + version='2012.1', description="Authentication service for OpenStack", author='OpenStack, LLC.', author_email='openstack@lists.launchpad.net', From 2340dee20f614da92a3e3615d06dcd2a3c91dfdf Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 13:45:41 -0800 Subject: [PATCH 141/334] rename extras to metadata --- keystone/backends/kvs.py | 62 ++++++++++++------------- keystone/backends/pam.py | 6 +-- keystone/backends/sql/core.py | 82 +++++++++++++++++----------------- keystone/backends/templated.py | 2 +- keystone/catalog.py | 4 +- keystone/identity.py | 16 +++---- keystone/keystone_compat.py | 42 ++++++++--------- keystone/models.py | 10 ++--- keystone/service.py | 32 ++++++------- keystone/test.py | 17 +++---- tests/default_fixtures.py | 2 +- tests/test_backend.py | 26 +++++------ tests/test_identity_api.py | 18 ++++---- tests/test_keystoneclient.py | 2 +- tests/test_legacy_compat.py | 4 +- 15 files changed, 164 insertions(+), 161 deletions(-) diff --git a/keystone/backends/kvs.py b/keystone/backends/kvs.py index c15e449c80..5e3d00fc21 100644 --- a/keystone/backends/kvs.py +++ b/keystone/backends/kvs.py @@ -27,7 +27,7 @@ class KvsIdentity(object): """ user_ref = self.get_user(user_id) tenant_ref = None - extras_ref = None + metadata_ref = None if not user_ref or user_ref.get('password') != password: raise AssertionError('Invalid user / password') if tenant_id and tenant_id not in user_ref['tenants']: @@ -35,10 +35,10 @@ class KvsIdentity(object): tenant_ref = self.get_tenant(tenant_id) if tenant_ref: - extras_ref = self.get_extras(user_id, tenant_id) + metadata_ref = self.get_metadata(user_id, tenant_id) else: - extras_ref = {} - return (user_ref, tenant_ref, extras_ref) + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) def get_tenant(self, tenant_id): tenant_ref = self.db.get('tenant-%s' % tenant_id) @@ -56,8 +56,8 @@ class KvsIdentity(object): user_ref = self.db.get('user_name-%s' % user_name) return user_ref - def get_extras(self, user_id, tenant_id): - return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + def get_metadata(self, user_id, tenant_id): + return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) def get_role(self, role_id): role_ref = self.db.get('role-%s' % role_id) @@ -91,28 +91,28 @@ class KvsIdentity(object): return user_ref.get('tenants', []) def get_roles_for_user_and_tenant(self, user_id, tenant_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - return extras_ref.get('roles', []) + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - roles = set(extras_ref.get('roles', [])) + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) roles.add(role_id) - extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - roles = set(extras_ref.get('roles', [])) + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) roles.remove(role_id) - extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) # CRUD def create_user(self, id, user): @@ -159,16 +159,16 @@ class KvsIdentity(object): self.db.delete('tenant-%s' % id) return None - def create_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras + def create_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata - def update_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras + def update_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata - def delete_extras(self, user_id, tenant_id): - self.db.delete('extras-%s-%s' % (tenant_id, user_id)) + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) return None def create_role(self, id, role): @@ -219,7 +219,7 @@ class KvsCatalog(object): self.db = db # Public interface - def get_catalog(self, user_id, tenant_id, extras=None): + def get_catalog(self, user_id, tenant_id, metadata=None): return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) def get_service(self, service_id): diff --git a/keystone/backends/pam.py b/keystone/backends/pam.py index a685bd13db..d960660138 100644 --- a/keystone/backends/pam.py +++ b/keystone/backends/pam.py @@ -13,16 +13,16 @@ class PamIdentity(object): def authenticate(self, username, password, **kwargs): if pam.authenticate(username, password): - extras = {} + metadata = {} if username == 'root': - extras['is_admin'] == True + metadata['is_admin'] == True tenant = {'id': username, 'name': username} user = {'id': username, 'name': username} - return (tenant, user, extras) + return (tenant, user, metadata) def get_tenants(self, username): return [{'id': username, diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index 7f56bdc287..a9a95bbe08 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -129,10 +129,10 @@ class Role(Base, DictBase): name = sql.Column(sql.String(64)) -class Extras(Base, DictBase): - __tablename__ = 'extras' +class Metadata(Base, DictBase): + __tablename__ = 'metadata' #__table_args__ = ( - # sql.Index('idx_extras_usertenant', 'user', 'tenant'), + # sql.Index('idx_metadata_usertenant', 'user', 'tenant'), # ) user_id = sql.Column(sql.String(64), primary_key=True) @@ -207,7 +207,7 @@ class SqlIdentity(SqlBase): """ user_ref = self.get_user(user_id) tenant_ref = None - extras_ref = None + metadata_ref = None if not user_ref or user_ref.get('password') != password: raise AssertionError('Invalid user / password') @@ -218,10 +218,10 @@ class SqlIdentity(SqlBase): tenant_ref = self.get_tenant(tenant_id) print 'ETESTSET', tenant_ref if tenant_ref: - extras_ref = self.get_extras(user_id, tenant_id) + metadata_ref = self.get_metadata(user_id, tenant_id) else: - extras_ref = {} - return (user_ref, tenant_ref, extras_ref) + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) def get_tenant(self, tenant_id): session = self.get_session() @@ -251,13 +251,13 @@ class SqlIdentity(SqlBase): return return user_ref.to_dict() - def get_extras(self, user_id, tenant_id): + def get_metadata(self, user_id, tenant_id): session = self.get_session() - extras_ref = session.query(Extras)\ + metadata_ref = session.query(Metadata)\ .filter_by(user_id=user_id)\ .filter_by(tenant_id=tenant_id)\ .first() - return getattr(extras_ref, 'data', None) + return getattr(metadata_ref, 'data', None) def get_role(self, role_id): session = self.get_session() @@ -298,38 +298,38 @@ class SqlIdentity(SqlBase): return [x.tenant_id for x in membership_refs] def get_roles_for_user_and_tenant(self, user_id, tenant_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - return extras_ref.get('roles', []) + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) + metadata_ref = self.get_metadata(user_id, tenant_id) is_new = False - if not extras_ref: + if not metadata_ref: is_new = True - extras_ref = {} - roles = set(extras_ref.get('roles', [])) + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) roles.add(role_id) - extras_ref['roles'] = list(roles) + metadata_ref['roles'] = list(roles) if not is_new: - self.update_extras(user_id, tenant_id, extras_ref) + self.update_metadata(user_id, tenant_id, metadata_ref) else: - self.create_extras(user_id, tenant_id, extras_ref) + self.create_metadata(user_id, tenant_id, metadata_ref) def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) + metadata_ref = self.get_metadata(user_id, tenant_id) is_new = False - if not extras_ref: + if not metadata_ref: is_new = True - extras_ref = {} - roles = set(extras_ref.get('roles', [])) + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) roles.remove(role_id) - extras_ref['roles'] = list(roles) + metadata_ref['roles'] = list(roles) if not is_new: - self.update_extras(user_id, tenant_id, extras_ref) + self.update_metadata(user_id, tenant_id, metadata_ref) else: - self.create_extras(user_id, tenant_id, extras_ref) + self.create_metadata(user_id, tenant_id, metadata_ref) # CRUD def create_user(self, id, user): @@ -384,27 +384,29 @@ class SqlIdentity(SqlBase): with session.begin(): session.delete(tenant_ref) - def create_extras(self, user_id, tenant_id, extras): + def create_metadata(self, user_id, tenant_id, metadata): session = self.get_session() with session.begin(): - session.add(Extras(user_id=user_id, tenant_id=tenant_id, data=extras)) - return extras + session.add(Metadata(user_id=user_id, + tenant_id=tenant_id, + data=metadata)) + return metadata - def update_extras(self, user_id, tenant_id, extras): + def update_metadata(self, user_id, tenant_id, metadata): session = self.get_session() with session.begin(): - extras_ref = session.query(Extras)\ + metadata_ref = session.query(Metadata)\ .filter_by(user_id=user_id)\ .filter_by(tenant_id=tenant_id)\ .first() - data = extras_ref.data.copy() - for k in extras: - data[k] = extras[k] - extras_ref.data = data - return extras_ref + data = metadata_ref.data.copy() + for k in metadata: + data[k] = metadata[k] + metadata_ref.data = data + return metadata_ref - def delete_extras(self, user_id, tenant_id): - self.db.delete('extras-%s-%s' % (tenant_id, user_id)) + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) return None def create_role(self, id, role): diff --git a/keystone/backends/templated.py b/keystone/backends/templated.py index 8f00fc41bd..f480459e99 100644 --- a/keystone/backends/templated.py +++ b/keystone/backends/templated.py @@ -70,7 +70,7 @@ class TemplatedCatalog(kvs.KvsCatalog): self.templates = o - def get_catalog(self, user_id, tenant_id, extras=None): + def get_catalog(self, user_id, tenant_id, metadata=None): d = dict(CONF.iteritems()) d.update({'tenant_id': tenant_id, 'user_id': user_id}) diff --git a/keystone/catalog.py b/keystone/catalog.py index 96cfd4586e..6ad348e5b9 100644 --- a/keystone/catalog.py +++ b/keystone/catalog.py @@ -13,9 +13,9 @@ class Manager(object): def __init__(self): self.driver = utils.import_object(CONF.catalog.driver) - def get_catalog(self, context, user_id, tenant_id, extras=None): + def get_catalog(self, context, user_id, tenant_id, metadata=None): """Return info for a catalog if it is valid.""" - return self.driver.get_catalog(user_id, tenant_id, extras=extras) + return self.driver.get_catalog(user_id, tenant_id, metadata=metadata) def get_service(self, context, service_id): return self.driver.get_service(service_id) diff --git a/keystone/identity.py b/keystone/identity.py index 6e606949df..60bada9f3f 100644 --- a/keystone/identity.py +++ b/keystone/identity.py @@ -32,8 +32,8 @@ class Manager(object): def get_tenant_by_name(self, context, tenant_name): return self.driver.get_tenant_by_name(tenant_name) - def get_extras(self, context, user_id, tenant_id): - return self.driver.get_extras(user_id, tenant_id) + def get_metadata(self, context, user_id, tenant_id): + return self.driver.get_metadata(user_id, tenant_id) def get_role(self, context, role_id): return self.driver.get_role(role_id) @@ -86,14 +86,14 @@ class Manager(object): def delete_tenant(self, context, tenant_id): return self.driver.delete_tenant(tenant_id) - def create_extras(self, context, user_id, tenant_id, data): - return self.driver.create_extras(user_id, tenant_id, data) + def create_metadata(self, context, user_id, tenant_id, data): + return self.driver.create_metadata(user_id, tenant_id, data) - def update_extras(self, context, user_id, tenant_id, data): - return self.driver.update_extras(user_id, tenant_id, data) + def update_metadata(self, context, user_id, tenant_id, data): + return self.driver.update_metadata(user_id, tenant_id, data) - def delete_extras(self, context, user_id, tenant_id): - return self.driver.delete_extras(user_id, tenant_id) + def delete_metadata(self, context, user_id, tenant_id): + return self.driver.delete_metadata(user_id, tenant_id) def create_role(self, context, role_id, data): return self.driver.create_role(role_id, data) diff --git a/keystone/keystone_compat.py b/keystone/keystone_compat.py index f87c8d46ac..e5748f3c45 100644 --- a/keystone/keystone_compat.py +++ b/keystone/keystone_compat.py @@ -319,22 +319,22 @@ class KeystoneTokenController(service.BaseApplication): else: tenant_id = auth.get('tenantId', None) - (user_ref, tenant_ref, extras_ref) = \ + (user_ref, tenant_ref, metadata_ref) = \ self.identity_api.authenticate(context=context, user_id=user_id, password=password, tenant_id=tenant_id) - token_ref = self.token_api.create_token(context, - dict(expires='', - user=user_ref, - tenant=tenant_ref, - extras=extras_ref)) + token_ref = self.token_api.create_token( + context, dict(expires='', + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) if tenant_ref: catalog_ref = self.catalog_api.get_catalog( context=context, user_id=user_ref['id'], tenant_id=tenant_ref['id'], - extras=extras_ref) + metadata=metadata_ref) else: catalog_ref = {} @@ -359,26 +359,26 @@ class KeystoneTokenController(service.BaseApplication): tenant_ref = self.identity_api.get_tenant(context=context, tenant_id=tenant_id) - extras_ref = self.identity_api.get_extras( + metadata_ref = self.identity_api.get_metadata( context=context, user_id=user_ref['id'], tenant_id=tenant_ref['id']) - token_ref = self.token_api.create_token(context, - dict(expires='', - user=user_ref, - tenant=tenant_ref, - extras=extras_ref)) + token_ref = self.token_api.create_token( + context, dict(expires='', + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) catalog_ref = self.catalog_api.get_catalog( context=context, user_id=user_ref['id'], tenant_id=tenant_ref['id'], - extras=extras_ref) + metadata=metadata_ref) # TODO(termie): optimize this call at some point and put it into the - # the return for extras - # fill out the roles in the extras + # the return for metadata + # fill out the roles in the metadata roles_ref = [] - for role_id in extras_ref.get('roles', []): + for role_id in metadata_ref.get('roles', []): roles_ref.append(self.identity_api.get_role(context, role_id)) logging.debug('TOKEN_REF %s', token_ref) return self._format_authenticate(token_ref, roles_ref, catalog_ref) @@ -397,7 +397,7 @@ class KeystoneTokenController(service.BaseApplication): if not context['is_admin']: user_token_ref = self.token_api.get_token( context=context, token_id=context['token_id']) - creds = user_token_ref['extras'].copy() + creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') # Accept either is_admin or the admin role @@ -427,7 +427,7 @@ class KeystoneTokenController(service.BaseApplication): def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] - extras_ref = token_ref['extras'] + metadata_ref = token_ref['metadata'] o = {'access': {'token': {'id': token_ref['id'], 'expires': token_ref['expires'] }, @@ -435,7 +435,7 @@ class KeystoneTokenController(service.BaseApplication): 'name': user_ref['name'], 'username': user_ref['name'], 'roles': roles_ref, - 'roles_links': extras_ref.get('roles_links', + 'roles_links': metadata_ref.get('roles_links', []) } } @@ -525,7 +525,7 @@ class KeystoneTenantController(service.BaseApplication): if not context['is_admin']: user_token_ref = self.token_api.get_token( context=context, token_id=context['token_id']) - creds = user_token_ref['extras'].copy() + creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') # Accept either is_admin or the admin role diff --git a/keystone/models.py b/keystone/models.py index 7cfec85c11..18402bbd3e 100644 --- a/keystone/models.py +++ b/keystone/models.py @@ -20,9 +20,9 @@ class Role(dict): super(Role, self).__init__(id=id, *args, **kw) -class Extras(dict): +class Metadata(dict): def __init__(self, user_id=None, tenant_id=None, *args, **kw): - super(Extras, self).__init__(user_id=user_id, - tenant_id=tenant_id, - *args, - **kw) + super(Metadata, self).__init__(user_id=user_id, + tenant_id=tenant_id, + *args, + **kw) diff --git a/keystone/service.py b/keystone/service.py index fa77830496..636b49dc38 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -17,7 +17,7 @@ HIGH_LEVEL_CALLS = { 'get_user': ('GET', '/user/%(user_id)s'), 'get_tenant': ('GET', '/tenant/%(tenant_id)s'), 'get_tenant_by_name': ('GET', '/tenant_name/%(tenant_name)s'), - 'get_extras': ('GET', '/extras/%(tenant_id)s-%(user_id)s'), + 'get_metadata': ('GET', '/metadata/%(tenant_id)s-%(user_id)s'), 'get_token': ('GET', '/token/%(token_id)s'), } @@ -34,11 +34,11 @@ LOW_LEVEL_CALLS = { 'create_tenant': ('POST', '/tenant'), 'update_tenant': ('PUT', '/tenant/%(tenant_id)s'), 'delete_tenant': ('DELETE', '/tenant/%(tenant_id)s'), - # extras + # metadata # NOTE(termie): these separators are probably going to bite us eventually - 'create_extras': ('POST', '/extras'), - 'update_extras': ('PUT', '/extras/%(tenant_id)s-%(user_id)s'), - 'delete_extras': ('DELETE', '/extras/%(tenant_id)s-%(user_id)s'), + 'create_metadata': ('POST', '/metadata'), + 'update_metadata': ('PUT', '/metadata/%(tenant_id)s-%(user_id)s'), + 'delete_metadata': ('DELETE', '/metadata/%(tenant_id)s-%(user_id)s'), } @@ -98,7 +98,7 @@ class BaseApplication(wsgi.Application): if not context['is_admin']: user_token_ref = self.token_api.get_token( context=context, token_id=context['token_id']) - creds = user_token_ref['extras'].copy() + creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') print creds @@ -136,13 +136,13 @@ class IdentityController(BaseApplication): return '' def authenticate(self, context, **kwargs): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( context, **kwargs) # TODO(termie): strip password from return values token_ref = self.token_api.create_token(context, dict(tenant=tenant_ref, user=user_ref, - extras=extras_ref)) + metadata=metadata_ref)) logging.debug('TOKEN: %s', token_ref) return token_ref @@ -197,24 +197,24 @@ class IdentityController(BaseApplication): def delete_tenant(self, context, tenant_id): return self.identity_api.delete_tenant(context, tenant_id=tenant_id) - def get_extras(self, context, user_id, tenant_id): - return self.identity_api.get_extras( + def get_metadata(self, context, user_id, tenant_id): + return self.identity_api.get_metadata( context, user_id=user_id, tenant_id=tenant_id) - def create_extras(self, context, **kw): + def create_metadata(self, context, **kw): user_id = kw.pop('user_id') tenant_id = kw.pop('tenant_id') - return self.identity_api.create_extras( + return self.identity_api.create_metadata( context, user_id=user_id, tenant_id=tenant_id, data=kw) - def update_extras(self, context, user_id, tenant_id, **kw): + def update_metadata(self, context, user_id, tenant_id, **kw): kw.pop('user_id', None) kw.pop('tenant_id', None) - return self.identity_api.update_extras( + return self.identity_api.update_metadata( context, user_id=user_id, tenant_id=tenant_id, data=kw) - def delete_extras(self, context, user_id, tenant_id): - return self.identity_api.delete_extras( + def delete_metadata(self, context, user_id, tenant_id): + return self.identity_api.delete_metadata( context, user_id=user_id, tenant_id=tenant_id) diff --git a/keystone/test.py b/keystone/test.py index bfd6a0ef4f..ccd43940cd 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -34,7 +34,7 @@ def checkout_vendor(repo, rev): try: if os.path.exists(modcheck): mtime = os.stat(modcheck).st_mtime - if int(time.time()) - mtime < 1000: + if int(time.time()) - mtime < 10000: return revdir if not os.path.exists(revdir): @@ -130,16 +130,17 @@ class TestCase(unittest.TestCase): rv = self.identity_api.create_role(role['id'], role) setattr(self, 'role_%s' % role['id'], rv) - for extras in fixtures.EXTRAS: - extras_ref = extras.copy() + for metadata in fixtures.METADATA: + metadata_ref = metadata.copy() # TODO(termie): these will probably end up in the model anyway, so this # may be futile - del extras_ref['user_id'] - del extras_ref['tenant_id'] - rv = self.identity_api.create_extras( - extras['user_id'], extras['tenant_id'], extras_ref) + del metadata_ref['user_id'] + del metadata_ref['tenant_id'] + rv = self.identity_api.create_metadata( + metadata['user_id'], metadata['tenant_id'], metadata_ref) setattr(self, - 'extras_%s%s' % (extras['user_id'], extras['tenant_id']), rv) + 'metadata_%s%s' % (metadata['user_id'], + metadata['tenant_id']), rv) def loadapp(self, config, name='main'): if not config.startswith('config:'): diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 0e24d750fc..786becd1f6 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -7,7 +7,7 @@ USERS = [ {'id': 'foo', 'name': 'FOO', 'password': 'foo2', 'tenants': ['bar',]}, ] -EXTRAS = [ +METADATA = [ {'user_id': 'foo', 'tenant_id': 'bar', 'extra': 'extra'}, ] diff --git a/tests/test_backend.py b/tests/test_backend.py index 31c37a2f70..1d4edd5b76 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -21,21 +21,21 @@ class IdentityTests(object): password=self.user_foo['password']) def test_authenticate_no_tenant(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( user_id=self.user_foo['id'], password=self.user_foo['password']) self.assertDictEquals(user_ref, self.user_foo) self.assert_(tenant_ref is None) - self.assert_(not extras_ref) + self.assert_(not metadata_ref) def test_authenticate(self): - user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id'], password=self.user_foo['password']) self.assertDictEquals(user_ref, self.user_foo) self.assertDictEquals(tenant_ref, self.tenant_bar) - self.assertDictEquals(extras_ref, self.extras_foobar) + self.assertDictEquals(metadata_ref, self.metadata_foobar) def test_get_tenant_bad_tenant(self): tenant_ref = self.identity_api.get_tenant( @@ -65,23 +65,23 @@ class IdentityTests(object): user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) self.assertDictEquals(user_ref, self.user_foo) - def test_get_extras_bad_user(self): - extras_ref = self.identity_api.get_extras( + def test_get_metadata_bad_user(self): + metadata_ref = self.identity_api.get_metadata( user_id=self.user_foo['id'] + 'WRONG', tenant_id=self.tenant_bar['id']) - self.assert_(extras_ref is None) + self.assert_(metadata_ref is None) - def test_get_extras_bad_tenant(self): - extras_ref = self.identity_api.get_extras( + def test_get_metadata_bad_tenant(self): + metadata_ref = self.identity_api.get_metadata( user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(extras_ref is None) + self.assert_(metadata_ref is None) - def test_get_extras(self): - extras_ref = self.identity_api.get_extras( + def test_get_metadata(self): + metadata_ref = self.identity_api.get_metadata( user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id']) - self.assertDictEquals(extras_ref, self.extras_foobar) + self.assertDictEquals(metadata_ref, self.metadata_foobar) def test_get_role(self): role_ref = self.identity_api.get_role( diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py index ef13ab1440..50d813cf64 100644 --- a/tests/test_identity_api.py +++ b/tests/test_identity_api.py @@ -38,7 +38,7 @@ class IdentityApi(test.TestCase): data = json.loads(resp.body) self.assertEquals(self.user_foo['id'], data['user']['id']) self.assertEquals(self.tenant_bar['id'], data['tenant']['id']) - self.assertDictEquals(self.extras_foobar, data['extras']) + self.assertDictEquals(self.metadata_foobar, data['metadata']) def test_authenticate_no_tenant(self): c = client.TestClient(self.app) @@ -48,7 +48,7 @@ class IdentityApi(test.TestCase): data = json.loads(resp.body) self.assertEquals(self.user_foo['id'], data['user']['id']) self.assertEquals(None, data['tenant']) - self.assertEquals({}, data['extras']) + self.assertEquals({}, data['metadata']) def test_get_tenants(self): token = self._login() @@ -131,32 +131,32 @@ class IdentityApi(test.TestCase): # TODO(termie): we should probably return not founds instead of None #self.assertEquals(delget_resp.status, '404 Not Found') - def test_crud_extras(self): + def test_crud_metadata(self): token_id = CONF.admin_token user_id = 'foo' tenant_id = 'bar' c = client.TestClient(self.app, token=token_id) - extras_ref = dict(baz='qaz') - resp = c.create_extras(user_id=user_id, tenant_id=tenant_id, **extras_ref) + metadata_ref = dict(baz='qaz') + resp = c.create_metadata(user_id=user_id, tenant_id=tenant_id, **metadata_ref) data = json.loads(resp.body) self.assertEquals(data['baz'], 'qaz') - get_resp = c.get_extras(user_id=user_id, tenant_id=tenant_id) + get_resp = c.get_metadata(user_id=user_id, tenant_id=tenant_id) get_data = json.loads(get_resp.body) self.assertDictEquals(data, get_data) - update_resp = c.update_extras(user_id=user_id, + update_resp = c.update_metadata(user_id=user_id, tenant_id=tenant_id, baz='WAZ') update_data = json.loads(update_resp.body) self.assertEquals('WAZ', update_data['baz']) - del_resp = c.delete_extras(user_id=user_id, tenant_id=tenant_id) + del_resp = c.delete_metadata(user_id=user_id, tenant_id=tenant_id) self.assertEquals(del_resp.body, '') - delget_resp = c.get_extras(user_id=user_id, tenant_id=tenant_id) + delget_resp = c.get_metadata(user_id=user_id, tenant_id=tenant_id) self.assertEquals(delget_resp.body, '') # TODO(termie): we should probably return not founds instead of None #self.assertEquals(delget_resp.status, '404 Not Found') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 3da7a77f40..fb464153be 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -57,7 +57,7 @@ class KcMasterTestCase(CompatTestCase): # is all working # TODO(termie): add an admin user to the fixtures and use that user # override the fixtures, for now - self.extras_foobar = self.identity_api.update_extras( + self.metadata_foobar = self.identity_api.update_metadata( self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index 1525acaabd..ae79a918f5 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -59,7 +59,7 @@ class CompatTestCase(test.TestCase): name='jqsmith', tenants=[self.tenant_345['id']], password='password')) - self.extras_123 = self.identity_api.create_extras( + self.metadata_123 = self.identity_api.create_metadata( self.user_123['id'], self.tenant_345['id'], dict(roles=[{'id': '234', 'name': 'compute:admin'}, @@ -73,7 +73,7 @@ class CompatTestCase(test.TestCase): expires='2010-11-01T03:32:15-05:00', user=self.user_123, tenant=self.tenant_345, - extras=self.extras_123)) + metadata=self.metadata_123)) # auth call # NOTE(termie): the service catalog in the sample doesn't really have From 8f46af011ea2c4756ac3d9e7d62fc6ce477f8566 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 14:32:02 -0800 Subject: [PATCH 142/334] basic service running again --- .gitignore | 4 ++- bin/keystone | 7 +++- etc/default.conf | 57 +++++++++++++++++++++++--------- etc/default_catalog.templates | 12 +++++++ etc/keystone.conf | 62 ++++++++++++++++++++++++++++++----- 5 files changed, 115 insertions(+), 27 deletions(-) create mode 100644 etc/default_catalog.templates diff --git a/.gitignore b/.gitignore index c20afefb53..4bc32265e5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ run_tests.log covhtml pep8.txt nosetests.xml -bla.db +tests/bla.db +docs/build +.DS_Store diff --git a/bin/keystone b/bin/keystone index 58e7b3a941..070608172d 100755 --- a/bin/keystone +++ b/bin/keystone @@ -18,9 +18,13 @@ if os.path.exists(os.path.join(possible_topdir, from paste import deploy +from keystone import config from keystone import wsgi +CONF = config.CONF + + def create_server(name, port): app = deploy.loadapp('config:%s' % conf, name=name) return wsgi.Server(app, port) @@ -40,7 +44,8 @@ def serve(*servers): if __name__ == '__main__': default_conf = os.path.join(possible_topdir, 'etc', - 'default.conf') + 'keystone.conf') + CONF(config_files=[default_conf]) logging.getLogger().setLevel(logging.DEBUG) conf = len(sys.argv) > 1 and sys.argv[1] or default_conf diff --git a/etc/default.conf b/etc/default.conf index f50c7706d6..3062f4b317 100644 --- a/etc/default.conf +++ b/etc/default.conf @@ -1,43 +1,68 @@ [DEFAULT] -catalog_driver = keystonelight.backends.kvs.KvsCatalog -identity_driver = keystonelight.backends.kvs.KvsIdentity -token_driver = keystonelight.backends.kvs.KvsToken -policy_driver = keystonelight.backends.policy.SimpleMatch public_port = 5000 admin_port = 35357 admin_token = ADMIN +compute_port = 3000 + +[sql] +connection = sqlite:///bla.db +idle_timeout = 200 +min_pool_size = 5 +max_pool_size = 10 +pool_timeout = 200 + +[identity] +driver = keystone.backends.kvs.KvsIdentity + +[catalog] +driver = keystone.backends.templated.TemplatedCatalog +template_file = default_catalog.templates + +[token] +driver = keystone.backends.kvs.KvsToken + +[policy] +driver = keystone.backends.policy.SimpleMatch + [filter:debug] -paste.filter_factory = keystonelight.wsgi:Debug.factory +paste.filter_factory = keystone.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.middleware:TokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.middleware:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.middleware:JsonBodyMiddleware.factory +paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory -[app:keystonelight] -paste.app_factory = keystonelight.service:app_factory +[filter:crud_extension] +paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory + +[app:keystone] +paste.app_factory = keystone.service:app_factory [app:keystone_service] -paste.app_factory = keystonelight.keystone_compat:service_app_factory +paste.app_factory = keystone.keystone_compat:service_app_factory [app:keystone_admin] -paste.app_factory = keystonelight.keystone_compat:admin_app_factory +paste.app_factory = keystone.keystone_compat:admin_app_factory -[pipeline:keystonelight_api] -pipeline = token_auth admin_token_auth json_body debug keystonelight +[pipeline:keystone_api] +pipeline = token_auth admin_token_auth json_body debug keystone [pipeline:keystone_service_api] pipeline = token_auth admin_token_auth json_body debug keystone_service [pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug keystone_admin +pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin [composite:main] use = egg:Paste#urlmap -/ = keystonelight_api +/ = keystone_api /v2.0 = keystone_service_api + +[composite:admin] +use = egg:Paste#urlmap +/v2.0 = keystone_admin_api diff --git a/etc/default_catalog.templates b/etc/default_catalog.templates new file mode 100644 index 0000000000..c12b5c4ca7 --- /dev/null +++ b/etc/default_catalog.templates @@ -0,0 +1,12 @@ +# config for TemplatedCatalog, using camelCase because I don't want to do +# translations for keystone compat +catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 +catalog.RegionOne.identity.adminURL = http://localhost:$(admin_port)s/v2.0 +catalog.RegionOne.identity.internalURL = http://localhost:$(admin_port)s/v2.0 +catalog.RegionOne.identity.name = 'Identity Service' + +# fake compute service for now to help novaclient tests work +catalog.RegionOne.compute.publicURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.adminURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$(tenant_id)s +catalog.RegionOne.compute.name = 'Compute Service' diff --git a/etc/keystone.conf b/etc/keystone.conf index d90526316e..bbeb1ca590 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -1,24 +1,68 @@ [DEFAULT] -catalog_driver = keystonelight.backends.kvs.KvsCatalog -identity_driver = keystonelight.backends.kvs.KvsIdentity -token_driver = keystonelight.backends.kvs.KvsToken public_port = 5000 +admin_port = 35357 admin_token = ADMIN +compute_port = 3000 + +[sql] +connection = sqlite:///bla.db +idle_timeout = 200 +min_pool_size = 5 +max_pool_size = 10 +pool_timeout = 200 + +[identity] +driver = keystone.backends.kvs.KvsIdentity + +[catalog] +driver = keystone.backends.templated.TemplatedCatalog +template_file = ./etc/default_catalog.templates + +[token] +driver = keystone.backends.kvs.KvsToken + +[policy] +driver = keystone.backends.policy.SimpleMatch + [filter:debug] -paste.filter_factory = keystonelight.wsgi:Debug.factory +paste.filter_factory = keystone.wsgi:Debug.factory [filter:token_auth] -paste.filter_factory = keystonelight.service:TokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory [filter:admin_token_auth] -paste.filter_factory = keystonelight.service:AdminTokenAuthMiddleware.factory +paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] -paste.filter_factory = keystonelight.service:JsonBodyMiddleware.factory +paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory + +[filter:crud_extension] +paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory [app:keystone] -paste.app_factory = keystonelight.keystone_compat:app_factory +paste.app_factory = keystone.service:app_factory -[pipeline:main] +[app:keystone_service] +paste.app_factory = keystone.keystone_compat:service_app_factory + +[app:keystone_admin] +paste.app_factory = keystone.keystone_compat:admin_app_factory + +[pipeline:keystone_api] pipeline = token_auth admin_token_auth json_body debug keystone + +[pipeline:keystone_service_api] +pipeline = token_auth admin_token_auth json_body debug keystone_service + +[pipeline:keystone_admin_api] +pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin + +[composite:main] +use = egg:Paste#urlmap +/ = keystone_api +/v2.0 = keystone_service_api + +[composite:admin] +use = egg:Paste#urlmap +/v2.0 = keystone_admin_api From 51df8b1b5ba2a71c0b2bd4e0b6a6a0a025576f76 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 14:34:28 -0800 Subject: [PATCH 143/334] remove default configuration --- etc/default.conf | 68 ------------------------------------------------ 1 file changed, 68 deletions(-) delete mode 100644 etc/default.conf diff --git a/etc/default.conf b/etc/default.conf deleted file mode 100644 index 3062f4b317..0000000000 --- a/etc/default.conf +++ /dev/null @@ -1,68 +0,0 @@ -[DEFAULT] -public_port = 5000 -admin_port = 35357 -admin_token = ADMIN -compute_port = 3000 - -[sql] -connection = sqlite:///bla.db -idle_timeout = 200 -min_pool_size = 5 -max_pool_size = 10 -pool_timeout = 200 - -[identity] -driver = keystone.backends.kvs.KvsIdentity - -[catalog] -driver = keystone.backends.templated.TemplatedCatalog -template_file = default_catalog.templates - -[token] -driver = keystone.backends.kvs.KvsToken - -[policy] -driver = keystone.backends.policy.SimpleMatch - - -[filter:debug] -paste.filter_factory = keystone.wsgi:Debug.factory - -[filter:token_auth] -paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory - -[filter:admin_token_auth] -paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory - -[filter:json_body] -paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory - -[filter:crud_extension] -paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory - -[app:keystone] -paste.app_factory = keystone.service:app_factory - -[app:keystone_service] -paste.app_factory = keystone.keystone_compat:service_app_factory - -[app:keystone_admin] -paste.app_factory = keystone.keystone_compat:admin_app_factory - -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone - -[pipeline:keystone_service_api] -pipeline = token_auth admin_token_auth json_body debug keystone_service - -[pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin - -[composite:main] -use = egg:Paste#urlmap -/ = keystone_api -/v2.0 = keystone_service_api - -[composite:admin] -use = egg:Paste#urlmap -/v2.0 = keystone_admin_api From 75e781a11918662af1a4f88091856af9d00cdd13 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 14:51:26 -0800 Subject: [PATCH 144/334] remove keystone from names, remove service --- keystone/keystone_compat.py | 148 ++++++++++++++------- keystone/service.py | 254 ------------------------------------ keystone/utils.py | 9 ++ tests/default.conf | 2 +- 4 files changed, 113 insertions(+), 300 deletions(-) delete mode 100644 keystone/service.py diff --git a/keystone/keystone_compat.py b/keystone/keystone_compat.py index e5748f3c45..95e054009e 100644 --- a/keystone/keystone_compat.py +++ b/keystone/keystone_compat.py @@ -1,28 +1,85 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # this is the web service frontend that emulates keystone -import logging +import json import urllib import urlparse import uuid import routes -from webob import exc +import webob.dec +import webob.exc from keystone import catalog from keystone import identity +from keystone import logging from keystone import policy from keystone import service from keystone import token +from keystone import utils from keystone import wsgi -class KeystoneAdminRouter(wsgi.Router): +class Application(wsgi.Application): + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + del arg_dict['controller'] + logging.debug('arg_dict: %s', arg_dict) + + context = req.environ.get('openstack.context', {}) + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + params.update(arg_dict) + + # TODO(termie): do some basic normalization on methods + method = getattr(self, action) + + # NOTE(vish): make sure we have no unicode keys for py2.6. + params = self._normalize_dict(params) + result = method(context, **params) + + if result is None or type(result) is str or type(result) is unicode: + return result + elif isinstance(result, webob.exc.WSGIHTTPException): + return result + + return self._serialize(result) + + def _serialize(self, result): + return json.dumps(result, cls=utils.SmarterEncoder) + + def _normalize_arg(self, arg): + return str(arg).replace(':', '_').replace('-', '_') + + def _normalize_dict(self, d): + return dict([(self._normalize_arg(k), v) + for (k, v) in d.iteritems()]) + + def assert_admin(self, context): + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['metadata'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + print creds + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + + +class AdminRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() # Token Operations - auth_controller = KeystoneTokenController() + auth_controller = TokenController() mapper.connect('/tokens', controller=auth_controller, action='authenticate', @@ -37,7 +94,7 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # Tenant Operations - tenant_controller = KeystoneTenantController() + tenant_controller = TenantController() mapper.connect('/tenants', controller=tenant_controller, action='get_tenants_for_token', @@ -48,14 +105,14 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # User Operations - user_controller = KeystoneUserController() + user_controller = UserController() mapper.connect('/users/{user_id}', controller=user_controller, action='get_user', conditions=dict(method=['GET'])) # Role Operations - roles_controller = KeystoneRoleController() + roles_controller = RoleController() mapper.connect('/tenants/{tenant_id}/users/{user_id}/roles', controller=roles_controller, action='get_user_roles', @@ -66,27 +123,27 @@ class KeystoneAdminRouter(wsgi.Router): conditions=dict(method=['GET'])) # Miscellaneous Operations - version_controller = KeystoneVersionController() + version_controller = VersionController() mapper.connect('/', controller=version_controller, action='get_version_info', module='admin/version', conditions=dict(method=['GET'])) - extensions_controller = KeystoneExtensionsController() + extensions_controller = ExtensionsController() mapper.connect('/extensions', controller=extensions_controller, action='get_extensions_info', conditions=dict(method=['GET'])) - super(KeystoneAdminRouter, self).__init__(mapper) + super(AdminRouter, self).__init__(mapper) -class KeystoneServiceRouter(wsgi.Router): +class ServiceRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() # Token Operations - auth_controller = KeystoneTokenController() + auth_controller = TokenController() mapper.connect('/tokens', controller=auth_controller, action='authenticate', @@ -97,30 +154,30 @@ class KeystoneServiceRouter(wsgi.Router): conditions=dict(methods=['POST'])) # Tenant Operations - tenant_controller = KeystoneTenantController() + tenant_controller = TenantController() mapper.connect('/tenants', controller=tenant_controller, action='get_tenants_for_token', conditions=dict(methods=['GET'])) # Miscellaneous - version_controller = KeystoneVersionController() + version_controller = VersionController() mapper.connect('/', controller=version_controller, action='get_version_info', module='service/version', conditions=dict(method=['GET'])) - extensions_controller = KeystoneExtensionsController() + extensions_controller = ExtensionsController() mapper.connect('/extensions', controller=extensions_controller, action='get_extensions_info', conditions=dict(method=['GET'])) - super(KeystoneServiceRouter, self).__init__(mapper) + super(ServiceRouter, self).__init__(mapper) -class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): +class AdminCrudExtension(wsgi.ExtensionRouter): """Previously known as the OS-KSADM extension. Provides a bunch of CRUD operations for internal data types. @@ -129,10 +186,10 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): def __init__(self, application): mapper = routes.Mapper() - tenant_controller = KeystoneTenantController() - user_controller = KeystoneUserController() - role_controller = KeystoneRoleController() - service_controller = KeystoneServiceController() + tenant_controller = TenantController() + user_controller = UserController() + role_controller = RoleController() + service_controller = ServiceController() # Tenant Operations mapper.connect("/tenants", controller=tenant_controller, @@ -265,22 +322,22 @@ class KeystoneAdminCrudExtension(wsgi.ExtensionRouter): action="delete_role", conditions=dict(method=["DELETE"])) - super(KeystoneAdminCrudExtension, self).__init__( + super(AdminCrudExtension, self).__init__( application, mapper) -class KeystoneTokenController(service.BaseApplication): +class TokenController(Application): def __init__(self): self.catalog_api = catalog.Manager() self.identity_api = identity.Manager() self.token_api = token.Manager() self.policy_api = policy.Manager() - super(KeystoneTokenController, self).__init__() + super(TokenController, self).__init__() def authenticate(self, context, auth=None): """Authenticate credentials and return a token. - Keystone accepts auth as a dict that looks like: + Accept auth as a dict that looks like: { "auth":{ @@ -446,7 +503,8 @@ class KeystoneTokenController(service.BaseApplication): return o def _format_catalog(self, catalog_ref): - """KeystoneLight catalogs look like: + """Munge catalogs from internal to output format + Internal catalogs look like: {$REGION: { {$SERVICE: { @@ -456,7 +514,7 @@ class KeystoneTokenController(service.BaseApplication): } } - Keystone's look like + The legacy api wants them to look like [{'name': $SERVICE[name], 'type': $SERVICE, @@ -490,12 +548,12 @@ class KeystoneTokenController(service.BaseApplication): return services.values() -class KeystoneTenantController(service.BaseApplication): +class TenantController(Application): def __init__(self): self.identity_api = identity.Manager() self.policy_api = policy.Manager() self.token_api = token.Manager() - super(KeystoneTenantController, self).__init__() + super(TenantController, self).__init__() def get_tenants_for_token(self, context, **kw): """Get valid tenants for token based on token used to authenticate. @@ -535,7 +593,7 @@ class KeystoneTenantController(service.BaseApplication): tenant = self.identity_api.get_tenant(context, tenant_id) if not tenant: - return exc.HTTPNotFound() + return webob.exc.HTTPNotFound() return {'tenant': tenant} # CRUD Extension @@ -573,19 +631,19 @@ class KeystoneTenantController(service.BaseApplication): return o -class KeystoneUserController(service.BaseApplication): +class UserController(Application): def __init__(self): self.catalog_api = catalog.Manager() self.identity_api = identity.Manager() self.policy_api = policy.Manager() self.token_api = token.Manager() - super(KeystoneUserController, self).__init__() + super(UserController, self).__init__() def get_user(self, context, user_id): self.assert_admin(context) user_ref = self.identity_api.get_user(context, user_id) if not user_ref: - raise exc.HTTPNotFound() + raise webob.exc.HTTPNotFound() return {'user': user_ref} def get_users(self, context): @@ -636,13 +694,13 @@ class KeystoneUserController(service.BaseApplication): return self.update_user(context, user_id, user) -class KeystoneRoleController(service.BaseApplication): +class RoleController(Application): def __init__(self): self.catalog_api = catalog.Manager() self.identity_api = identity.Manager() self.token_api = token.Manager() self.policy_api = policy.Manager() - super(KeystoneRoleController, self).__init__() + super(RoleController, self).__init__() def get_user_roles(self, context, user_id, tenant_id=None): raise NotImplemented() @@ -652,7 +710,7 @@ class KeystoneRoleController(service.BaseApplication): self.assert_admin(context) role_ref = self.identity_api.get_role(context, role_id) if not role_ref: - raise exc.HTTPNotFound() + raise webob.exc.HTTPNotFound() return {'role': role_ref} def create_role(self, context, role): @@ -740,13 +798,13 @@ class KeystoneRoleController(service.BaseApplication): context, tenant_id, user_id) -class KeystoneServiceController(service.BaseApplication): +class ServiceController(Application): def __init__(self): self.catalog_api = catalog.Manager() self.identity_api = identity.Manager() self.token_api = token.Manager() self.policy_api = policy.Manager() - super(KeystoneServiceController, self).__init__() + super(ServiceController, self).__init__() # CRUD extensions # NOTE(termie): this OS-KSADM stuff is about the lamest ever @@ -759,7 +817,7 @@ class KeystoneServiceController(service.BaseApplication): def get_service(self, context, service_id): service_ref = self.catalog_api.get_service(context, service_id) if not service_ref: - raise exc.HTTPNotFound() + raise webob.exc.HTTPNotFound() return {'OS-KSADM:service': service_ref} def delete_service(self, context, service_id): @@ -774,17 +832,17 @@ class KeystoneServiceController(service.BaseApplication): return {'OS-KSADM:service': new_service_ref} -class KeystoneVersionController(service.BaseApplication): +class VersionController(Application): def __init__(self): - super(KeystoneVersionController, self).__init__() + super(VersionController, self).__init__() def get_version_info(self, context, module='version'): raise NotImplemented() -class KeystoneExtensionsController(service.BaseApplication): +class ExtensionsController(Application): def __init__(self): - super(KeystoneExtensionsController, self).__init__() + super(ExtensionsController, self).__init__() def get_extensions_info(self, context): raise NotImplemented() @@ -793,10 +851,10 @@ class KeystoneExtensionsController(service.BaseApplication): def service_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return KeystoneServiceRouter() + return ServiceRouter() def admin_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return KeystoneAdminRouter() + return AdminRouter() diff --git a/keystone/service.py b/keystone/service.py deleted file mode 100644 index 636b49dc38..0000000000 --- a/keystone/service.py +++ /dev/null @@ -1,254 +0,0 @@ -import json -import logging -import uuid - -import routes -import webob.dec -import webob.exc - -from keystone import identity -from keystone import token -from keystone import wsgi - - -HIGH_LEVEL_CALLS = { - 'authenticate': ('POST', '/tokens'), - 'get_tenants': ('GET', '/user/%(user_id)s/tenants'), - 'get_user': ('GET', '/user/%(user_id)s'), - 'get_tenant': ('GET', '/tenant/%(tenant_id)s'), - 'get_tenant_by_name': ('GET', '/tenant_name/%(tenant_name)s'), - 'get_metadata': ('GET', '/metadata/%(tenant_id)s-%(user_id)s'), - 'get_token': ('GET', '/token/%(token_id)s'), - } - -# NOTE(termie): creates are seperate from updates to allow duplication checks -LOW_LEVEL_CALLS = { - # tokens - 'create_token': ('POST', '/token'), - 'delete_token': ('DELETE', '/token/%(token_id)s'), - # users - 'create_user': ('POST', '/user'), - 'update_user': ('PUT', '/user/%(user_id)s'), - 'delete_user': ('DELETE', '/user/%(user_id)s'), - # tenants - 'create_tenant': ('POST', '/tenant'), - 'update_tenant': ('PUT', '/tenant/%(tenant_id)s'), - 'delete_tenant': ('DELETE', '/tenant/%(tenant_id)s'), - # metadata - # NOTE(termie): these separators are probably going to bite us eventually - 'create_metadata': ('POST', '/metadata'), - 'update_metadata': ('PUT', '/metadata/%(tenant_id)s-%(user_id)s'), - 'delete_metadata': ('DELETE', '/metadata/%(tenant_id)s-%(user_id)s'), - } - - -URLMAP = HIGH_LEVEL_CALLS.copy() -URLMAP.update(LOW_LEVEL_CALLS) - - -class SmarterEncoder(json.JSONEncoder): - def default(self, obj): - if not isinstance(obj, dict) and hasattr(obj, 'iteritems'): - return dict(obj.iteritems()) - return super(SmarterEncoder, self).default(obj) - - -class BaseApplication(wsgi.Application): - @webob.dec.wsgify - def __call__(self, req): - arg_dict = req.environ['wsgiorg.routing_args'][1] - action = arg_dict['action'] - del arg_dict['action'] - del arg_dict['controller'] - logging.debug('arg_dict: %s', arg_dict) - - context = req.environ.get('openstack.context', {}) - # allow middleware up the stack to override the params - params = {} - if 'openstack.params' in req.environ: - params = req.environ['openstack.params'] - params.update(arg_dict) - - # TODO(termie): do some basic normalization on methods - method = getattr(self, action) - - # NOTE(vish): make sure we have no unicode keys for py2.6. - params = dict([(self._normalize_arg(k), v) - for (k, v) in params.iteritems()]) - result = method(context, **params) - - if result is None or type(result) is str or type(result) is unicode: - return result - elif isinstance(result, webob.exc.WSGIHTTPException): - return result - - return self._serialize(result) - - def _serialize(self, result): - return json.dumps(result, cls=SmarterEncoder) - - def _normalize_arg(self, arg): - return str(arg).replace(':', '_').replace('-', '_') - - def _normalize_dict(self, d): - return dict([(self._normalize_arg(k), v) - for (k, v) in d.iteritems()]) - - def assert_admin(self, context): - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['metadata'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - print creds - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) - - -class TokenController(BaseApplication): - """Validate and pass through calls to TokenManager.""" - - def __init__(self): - self.token_api = token.Manager() - - def validate_token(self, context, token_id): - token_info = self.token_api.validate_token(context, token_id) - if not token_info: - raise webob.exc.HTTPUnauthorized() - return token_info - - -class IdentityController(BaseApplication): - """Validate and pass calls through to IdentityManager. - - IdentityManager will also pretty much just pass calls through to - a specific driver. - """ - - def __init__(self): - self.identity_api = identity.Manager() - self.token_api = token.Manager() - - def noop(self, context, *args, **kw): - return '' - - def authenticate(self, context, **kwargs): - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - context, **kwargs) - # TODO(termie): strip password from return values - token_ref = self.token_api.create_token(context, - dict(tenant=tenant_ref, - user=user_ref, - metadata=metadata_ref)) - logging.debug('TOKEN: %s', token_ref) - return token_ref - - def get_tenants(self, context, user_id=None): - token_id = context.get('token_id') - token_ref = self.token_api.get_token(context, token_id) - assert token_ref - assert token_ref['user']['id'] == user_id - tenants_ref = [] - for tenant_id in token_ref['user']['tenants']: - tenants_ref.append(self.identity_api.get_tenant(context, - tenant_id)) - - return tenants_ref - - # crud api - def get_user(self, context, user_id): - return self.identity_api.get_user(context, user_id=user_id) - - def create_user(self, context, **kw): - user_id = kw.get('id') and kw.get('id') or uuid.uuid4().hex - kw['id'] = user_id - return self.identity_api.create_user(context, user_id=user_id, data=kw) - - def update_user(self, context, user_id, **kw): - kw['id'] = user_id - kw.pop('user_id', None) - return self.identity_api.update_user(context, user_id=user_id, data=kw) - - def delete_user(self, context, user_id): - return self.identity_api.delete_user(context, user_id=user_id) - - def get_tenant(self, context, tenant_id): - return self.identity_api.get_tenant(context, tenant_id=tenant_id) - - def get_tenant_by_name(self, context, tenant_name): - return self.identity_api.get_tenant_by_name( - context, tenant_name=tenant_name) - - def create_tenant(self, context, **kw): - tenant_id = kw.get('id') and kw.get('id') or uuid.uuid4().hex - kw['id'] = tenant_id - return self.identity_api.create_tenant( - context, tenant_id=tenant_id, data=kw) - - def update_tenant(self, context, tenant_id, **kw): - kw['id'] = tenant_id - kw.pop('tenant_id', None) - return self.identity_api.update_tenant( - context, tenant_id=tenant_id, data=kw) - - def delete_tenant(self, context, tenant_id): - return self.identity_api.delete_tenant(context, tenant_id=tenant_id) - - def get_metadata(self, context, user_id, tenant_id): - return self.identity_api.get_metadata( - context, user_id=user_id, tenant_id=tenant_id) - - def create_metadata(self, context, **kw): - user_id = kw.pop('user_id') - tenant_id = kw.pop('tenant_id') - return self.identity_api.create_metadata( - context, user_id=user_id, tenant_id=tenant_id, data=kw) - - def update_metadata(self, context, user_id, tenant_id, **kw): - kw.pop('user_id', None) - kw.pop('tenant_id', None) - return self.identity_api.update_metadata( - context, user_id=user_id, tenant_id=tenant_id, data=kw) - - def delete_metadata(self, context, user_id, tenant_id): - return self.identity_api.delete_metadata( - context, user_id=user_id, tenant_id=tenant_id) - - -class Router(wsgi.Router): - def __init__(self): - self.identity_controller = IdentityController() - self.token_controller = TokenController() - - mapper = self._build_map(URLMAP) - mapper.connect('/', controller=self.identity_controller, action='noop') - super(Router, self).__init__(mapper) - - def _build_map(self, urlmap): - """Build a routes.Mapper based on URLMAP.""" - mapper = routes.Mapper() - for k, v in urlmap.iteritems(): - # NOTE(termie): hack - if 'token' in k: - controller = self.token_controller - else: - controller = self.identity_controller - action = k - method, path = v - path = path.replace('%(', '{').replace(')s', '}') - - mapper.connect(path, - controller=controller, - action=action, - conditions=dict(method=[method])) - - return mapper - - -def app_factory(global_conf, **local_conf): - #conf = global_conf.copy() - #conf.update(local_conf) - return Router() diff --git a/keystone/utils.py b/keystone/utils.py index 139c563122..9679dde249 100644 --- a/keystone/utils.py +++ b/keystone/utils.py @@ -17,6 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. +import json import subprocess import sys @@ -44,6 +45,14 @@ def import_object(import_str, *args, **kw): return cls(*args, **kw) +class SmarterEncoder(json.JSONEncoder): + """Help for JSON encoding dict-like objects.""" + def default(self, obj): + if not isinstance(obj, dict) and hasattr(obj, 'iteritems'): + return dict(obj.iteritems()) + return super(SmarterEncoder, self).default(obj) + + # From python 2.7 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. diff --git a/tests/default.conf b/tests/default.conf index 3062f4b317..a92b39431c 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -38,7 +38,7 @@ paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] -paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory +paste.filter_factory = keystone.keystone_compat:AdminCrudExtension.factory [app:keystone] paste.app_factory = keystone.service:app_factory From ec82e9b1c9bc4da3631454b2077bbcec42003623 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 15:03:29 -0800 Subject: [PATCH 145/334] keystone_compat -> service --- keystone/client.py | 101 ------------ keystone/{keystone_compat.py => service.py} | 1 - run_tests.sh | 2 +- tests/default.conf | 12 +- tests/keystone_compat_diablo.conf | 14 +- tests/test_identity_api.py | 162 -------------------- 6 files changed, 11 insertions(+), 281 deletions(-) delete mode 100644 keystone/client.py rename keystone/{keystone_compat.py => service.py} (99%) delete mode 100644 tests/test_identity_api.py diff --git a/keystone/client.py b/keystone/client.py deleted file mode 100644 index 899e8d168b..0000000000 --- a/keystone/client.py +++ /dev/null @@ -1,101 +0,0 @@ - -"""Client library for KeystoneLight API.""" - -import json - -import httplib2 -import webob - -from keystone import service -from keystone import wsgi - - -URLMAP = service.URLMAP - - -class Client(object): - def __init__(self, token=None): - self.token = token - - def request(self, method, path, headers, body): - raise NotImplemented - - def get(self, path, headers=None): - return self.request('GET', path=path, headers=headers) - - def post(self, path, headers=None, body=None): - return self.request('POST', path=path, headers=headers, body=body) - - def put(self, path, headers=None, body=None): - return self.request('PUT', path=path, headers=headers, body=body) - - def _build_headers(self, headers=None): - if headers is None: - headers = {} - - if self.token: - headers.setdefault('X-Auth-Token', self.token) - - return headers - - def __getattr__(self, key): - """Lazy way to define a bunch of dynamic urls based on URLMAP. - - Turns something like - - c.authenticate(user_id='foo', password='bar') - - into - - c.request('POST', '/token', body={'user_id': 'foo', 'password': 'bar'}) - - """ - if key not in URLMAP: - raise AttributeError(key) - - method, path = URLMAP[key] - - def _internal(method_=method, path_=path, **kw): - path_ = path_ % kw - params = {'method': method_, - 'path': path_} - if method.lower() in ('put', 'post'): - params['body'] = kw - return self.request(**params) - - setattr(self, key, _internal) - - return getattr(self, key) - - -class HttpClient(Client): - def __init__(self, endpoint=None, token=None): - self.endpoint = endpoint - super(HttpClient, self).__init__(token=token) - - def request(self, method, path, headers=None, body=None): - if type(body) is type({}): - body = json.dumps(body) - headers = self._build_headers(headers) - h = httplib2.Http() - url = '%s%s' % (self.endpoint, path) - resp, content = h.request(url, method=method, headers=headers, body=body) - return webob.Response(content, status=resp.status, headerlist=resp.items()) - - -class TestClient(Client): - def __init__(self, app=None, token=None): - self.app = app - super(TestClient, self).__init__(token=token) - - def request(self, method, path, headers=None, body=None): - if type(body) is type({}): - body = json.dumps(body) - headers = self._build_headers(headers) - req = wsgi.Request.blank(path) - req.method = method - for k, v in headers.iteritems(): - req.headers[k] = v - if body: - req.body = body - return req.get_response(self.app) diff --git a/keystone/keystone_compat.py b/keystone/service.py similarity index 99% rename from keystone/keystone_compat.py rename to keystone/service.py index 95e054009e..7bdd5d26fd 100644 --- a/keystone/keystone_compat.py +++ b/keystone/service.py @@ -14,7 +14,6 @@ from keystone import catalog from keystone import identity from keystone import logging from keystone import policy -from keystone import service from keystone import token from keystone import utils from keystone import wsgi diff --git a/run_tests.sh b/run_tests.sh index be245a9afe..825d9501b1 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,7 +4,7 @@ set -eu function usage { echo "Usage: $0 [OPTION]..." - echo "Run KeystoneLight's test suite(s)" + echo "Run Keystone's test suite(s)" echo "" echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" diff --git a/tests/default.conf b/tests/default.conf index a92b39431c..f8771b9777 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -38,19 +38,14 @@ paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] -paste.filter_factory = keystone.keystone_compat:AdminCrudExtension.factory +paste.filter_factory = keystone.service:AdminCrudExtension.factory -[app:keystone] -paste.app_factory = keystone.service:app_factory [app:keystone_service] -paste.app_factory = keystone.keystone_compat:service_app_factory +paste.app_factory = keystone.service:service_app_factory [app:keystone_admin] -paste.app_factory = keystone.keystone_compat:admin_app_factory - -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +paste.app_factory = keystone.service:admin_app_factory [pipeline:keystone_service_api] pipeline = token_auth admin_token_auth json_body debug keystone_service @@ -60,7 +55,6 @@ pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_a [composite:main] use = egg:Paste#urlmap -/ = keystone_api /v2.0 = keystone_service_api [composite:admin] diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf index 842bb810dc..5db43f6f7c 100644 --- a/tests/keystone_compat_diablo.conf +++ b/tests/keystone_compat_diablo.conf @@ -28,17 +28,13 @@ paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory [filter:json_body] paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory -[app:keystone] -paste.app_factory = keystone.service:app_factory [app:keystone_service] -paste.app_factory = keystone.keystone_compat:service_app_factory +paste.app_factory = keystone.service:service_app_factory [app:keystone_admin] -paste.app_factory = keystone.keystone_compat:admin_app_factory +paste.app_factory = keystone.service:admin_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone [pipeline:keystone_service_api] pipeline = token_auth admin_token_auth json_body debug keystone_service @@ -46,7 +42,11 @@ pipeline = token_auth admin_token_auth json_body debug keystone_service [pipeline:keystone_admin_api] pipeline = token_auth admin_token_auth json_body debug keystone_admin + [composite:main] use = egg:Paste#urlmap -/ = keystone_api /v2.0 = keystone_service_api + +[composite:admin] +use = egg:Paste#urlmap +/v2.0 = keystone_admin_api diff --git a/tests/test_identity_api.py b/tests/test_identity_api.py deleted file mode 100644 index 50d813cf64..0000000000 --- a/tests/test_identity_api.py +++ /dev/null @@ -1,162 +0,0 @@ -import json - -from keystone import client -from keystone import config -from keystone import models -from keystone import test - -import default_fixtures - - -CONF = config.CONF - - -class IdentityApi(test.TestCase): - def setUp(self): - super(IdentityApi, self).setUp() - CONF(config_files=['default.conf']) - self.app = self.loadapp('default') - - self.load_backends() - self.load_fixtures(default_fixtures) - - def _login(self): - c = client.TestClient(self.app) - post_data = {'user_id': self.user_foo['id'], - 'tenant_id': self.tenant_bar['id'], - 'password': self.user_foo['password']} - resp = c.post('/tokens', body=post_data) - token = json.loads(resp.body) - return token - - def test_authenticate(self): - c = client.TestClient(self.app) - post_data = {'user_id': self.user_foo['id'], - 'tenant_id': self.tenant_bar['id'], - 'password': self.user_foo['password']} - resp = c.authenticate(**post_data) - data = json.loads(resp.body) - self.assertEquals(self.user_foo['id'], data['user']['id']) - self.assertEquals(self.tenant_bar['id'], data['tenant']['id']) - self.assertDictEquals(self.metadata_foobar, data['metadata']) - - def test_authenticate_no_tenant(self): - c = client.TestClient(self.app) - post_data = {'user_id': self.user_foo['id'], - 'password': self.user_foo['password']} - resp = c.authenticate(**post_data) - data = json.loads(resp.body) - self.assertEquals(self.user_foo['id'], data['user']['id']) - self.assertEquals(None, data['tenant']) - self.assertEquals({}, data['metadata']) - - def test_get_tenants(self): - token = self._login() - c = client.TestClient(self.app, token['id']) - resp = c.get_tenants(user_id=self.user_foo['id']) - data = json.loads(resp.body) - self.assertDictEquals(self.tenant_bar, data[0]) - - def test_crud_user(self): - token_id = CONF.admin_token - c = client.TestClient(self.app, token=token_id) - user_ref = models.User(name='FOO') - resp = c.create_user(**user_ref) - data = json.loads(resp.body) - self.assert_(data['id']) - - get_resp = c.get_user(user_id=data['id']) - get_data = json.loads(get_resp.body) - - self.assertDictEquals(data, get_data) - - update_resp = c.update_user(user_id=data['id'], - name='FOO', - id=data['id'], - password='foo') - update_data = json.loads(update_resp.body) - - self.assertEquals(data['id'], update_data['id']) - self.assertEquals('foo', update_data['password']) - - del_resp = c.delete_user(user_id=data['id']) - self.assertEquals(del_resp.body, '') - - delget_resp = c.get_user(user_id=data['id']) - self.assertEquals(delget_resp.body, '') - # TODO(termie): we should probably return not founds instead of None - #self.assertEquals(delget_resp.status, '404 Not Found') - - def test_crud_tenant(self): - token_id = CONF.admin_token - c = client.TestClient(self.app, token=token_id) - tenant_ref = models.Tenant(name='BAZ') - resp = c.create_tenant(**tenant_ref) - data = json.loads(resp.body) - self.assert_(data['id']) - - get_resp = c.get_tenant(tenant_id=data['id']) - get_data = json.loads(get_resp.body) - self.assertDictEquals(data, get_data) - - getname_resp = c.get_tenant_by_name(tenant_name=data['name']) - getname_data = json.loads(getname_resp.body) - self.assertDictEquals(data, getname_data) - - update_resp = c.update_tenant(tenant_id=data['id'], - id=data['id'], - name='NEWBAZ') - update_data = json.loads(update_resp.body) - - self.assertEquals(data['id'], update_data['id']) - self.assertEquals('NEWBAZ', update_data['name']) - - # make sure we can't get the old name - getname_resp = c.get_tenant_by_name(tenant_name=data['name']) - self.assertEquals(getname_resp.body, '') - - # but can get the new name - getname_resp = c.get_tenant_by_name(tenant_name=update_data['name']) - getname_data = json.loads(getname_resp.body) - self.assertDictEquals(update_data, getname_data) - - del_resp = c.delete_tenant(tenant_id=data['id']) - self.assertEquals(del_resp.body, '') - - delget_resp = c.get_tenant(tenant_id=data['id']) - self.assertEquals(delget_resp.body, '') - - delgetname_resp = c.get_tenant_by_name(tenant_name=update_data['name']) - self.assertEquals(delgetname_resp.body, '') - # TODO(termie): we should probably return not founds instead of None - #self.assertEquals(delget_resp.status, '404 Not Found') - - def test_crud_metadata(self): - token_id = CONF.admin_token - user_id = 'foo' - tenant_id = 'bar' - c = client.TestClient(self.app, token=token_id) - metadata_ref = dict(baz='qaz') - resp = c.create_metadata(user_id=user_id, tenant_id=tenant_id, **metadata_ref) - data = json.loads(resp.body) - self.assertEquals(data['baz'], 'qaz') - - get_resp = c.get_metadata(user_id=user_id, tenant_id=tenant_id) - get_data = json.loads(get_resp.body) - - self.assertDictEquals(data, get_data) - - update_resp = c.update_metadata(user_id=user_id, - tenant_id=tenant_id, - baz='WAZ') - update_data = json.loads(update_resp.body) - - self.assertEquals('WAZ', update_data['baz']) - - del_resp = c.delete_metadata(user_id=user_id, tenant_id=tenant_id) - self.assertEquals(del_resp.body, '') - - delget_resp = c.get_metadata(user_id=user_id, tenant_id=tenant_id) - self.assertEquals(delget_resp.body, '') - # TODO(termie): we should probably return not founds instead of None - #self.assertEquals(delget_resp.status, '404 Not Found') From a606c399665feb1c4e64053902659d4af7f396fb Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 15:14:05 -0800 Subject: [PATCH 146/334] rename many service parts to public --- keystone/service.py | 12 ++++++------ tests/default.conf | 18 +++++++++--------- tests/test_legacy_compat.py | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/keystone/service.py b/keystone/service.py index 7bdd5d26fd..190774f2bc 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# this is the web service frontend that emulates keystone +# this is the web public frontend that emulates keystone import json import urllib import urlparse @@ -137,7 +137,7 @@ class AdminRouter(wsgi.Router): super(AdminRouter, self).__init__(mapper) -class ServiceRouter(wsgi.Router): +class PublicRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() @@ -173,7 +173,7 @@ class ServiceRouter(wsgi.Router): action='get_extensions_info', conditions=dict(method=['GET'])) - super(ServiceRouter, self).__init__(mapper) + super(PublicRouter, self).__init__(mapper) class AdminCrudExtension(wsgi.ExtensionRouter): @@ -806,7 +806,7 @@ class ServiceController(Application): super(ServiceController, self).__init__() # CRUD extensions - # NOTE(termie): this OS-KSADM stuff is about the lamest ever + # NOTE(termie): this OS-KSADM stuff is not very consistent def get_services(self, context): service_list = self.catalog_api.list_services(context) service_refs = [self.catalog_api.get_service(context, x) @@ -847,10 +847,10 @@ class ExtensionsController(Application): raise NotImplemented() -def service_app_factory(global_conf, **local_conf): +def public_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - return ServiceRouter() + return PublicRouter() def admin_app_factory(global_conf, **local_conf): diff --git a/tests/default.conf b/tests/default.conf index f8771b9777..c2a716beb9 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -41,22 +41,22 @@ paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory paste.filter_factory = keystone.service:AdminCrudExtension.factory -[app:keystone_service] -paste.app_factory = keystone.service:service_app_factory +[app:public_service] +paste.app_factory = keystone.service:public_app_factory -[app:keystone_admin] +[app:admin_service] paste.app_factory = keystone.service:admin_app_factory -[pipeline:keystone_service_api] -pipeline = token_auth admin_token_auth json_body debug keystone_service +[pipeline:public_api] +pipeline = token_auth admin_token_auth json_body debug public_service -[pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin +[pipeline:admin_api] +pipeline = token_auth admin_token_auth json_body debug crud_extension admin_service [composite:main] use = egg:Paste#urlmap -/v2.0 = keystone_service_api +/v2.0 = public_api [composite:admin] use = egg:Paste#urlmap -/v2.0 = keystone_admin_api +/v2.0 = admin_api diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index ae79a918f5..765b9dab90 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -117,11 +117,11 @@ class CompatTestCase(test.TestCase): class DiabloCompatTestCase(CompatTestCase): def setUp(self): - CONF(config_files=['keystone_compat_diablo.conf']) + CONF(config_files=['default.conf']) revdir = test.checkout_vendor(KEYSTONE_REPO, 'stable/diablo') self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) - self.app = self.loadapp('keystone_compat_diablo') + self.app = self.loadapp('default') self.load_backends() super(DiabloCompatTestCase, self).setUp() From 74170ee7b4504748c5842a7935e64133970f587f Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 15:16:00 -0800 Subject: [PATCH 147/334] in the bin config too --- bin/keystone | 4 ++-- etc/keystone.conf | 28 +++++++++++----------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/bin/keystone b/bin/keystone index 070608172d..35514b6f6c 100755 --- a/bin/keystone +++ b/bin/keystone @@ -52,8 +52,8 @@ if __name__ == '__main__': options = deploy.appconfig('config:%s' % conf) servers = [] - servers.append(create_server('keystone_admin', + servers.append(create_server('admin', int(options['admin_port']))) - servers.append(create_server('keystone', + servers.append(create_server('main', int(options['public_port']))) serve(*servers) diff --git a/etc/keystone.conf b/etc/keystone.conf index bbeb1ca590..a1837095fb 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -38,31 +38,25 @@ paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] -paste.filter_factory = keystone.keystone_compat:KeystoneAdminCrudExtension.factory +paste.filter_factory = keystone.service:AdminCrudExtension.factory -[app:keystone] -paste.app_factory = keystone.service:app_factory -[app:keystone_service] -paste.app_factory = keystone.keystone_compat:service_app_factory +[app:public_service] +paste.app_factory = keystone.service:public_app_factory -[app:keystone_admin] -paste.app_factory = keystone.keystone_compat:admin_app_factory +[app:admin_service] +paste.app_factory = keystone.service:admin_app_factory -[pipeline:keystone_api] -pipeline = token_auth admin_token_auth json_body debug keystone +[pipeline:public_api] +pipeline = token_auth admin_token_auth json_body debug public_service -[pipeline:keystone_service_api] -pipeline = token_auth admin_token_auth json_body debug keystone_service - -[pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension keystone_admin +[pipeline:admin_api] +pipeline = token_auth admin_token_auth json_body debug crud_extension admin_service [composite:main] use = egg:Paste#urlmap -/ = keystone_api -/v2.0 = keystone_service_api +/v2.0 = public_api [composite:admin] use = egg:Paste#urlmap -/v2.0 = keystone_admin_api +/v2.0 = admin_api From 1d6334d3c25a815797fa040aa2ae3451ea3075d7 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 15:37:31 -0800 Subject: [PATCH 148/334] some more config in bin/keystone --- bin/keystone | 21 ++++++++++++++------- keystone/config.py | 5 ++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/bin/keystone b/bin/keystone index 35514b6f6c..881e517574 100755 --- a/bin/keystone +++ b/bin/keystone @@ -25,7 +25,7 @@ from keystone import wsgi CONF = config.CONF -def create_server(name, port): +def create_server(conf, name, port): app = deploy.loadapp('config:%s' % conf, name=name) return wsgi.Server(app, port) @@ -42,18 +42,25 @@ def serve(*servers): if __name__ == '__main__': - default_conf = os.path.join(possible_topdir, + dev_conf = os.path.join(possible_topdir, 'etc', 'keystone.conf') - CONF(config_files=[default_conf]) + config_files = None + if os.path.exists(dev_conf): + config_files = [dev_conf] + + CONF(config_files=config_files) + + # TODO(termie): configure this based on config logging.getLogger().setLevel(logging.DEBUG) - conf = len(sys.argv) > 1 and sys.argv[1] or default_conf - options = deploy.appconfig('config:%s' % conf) + options = deploy.appconfig('config:%s' % CONF.config_file[0]) servers = [] - servers.append(create_server('admin', + servers.append(create_server(CONF.config_file[0], + 'admin', int(options['admin_port']))) - servers.append(create_server('main', + servers.append(create_server(CONF.config_file[0], + 'main', int(options['public_port']))) serve(*servers) diff --git a/keystone/config.py b/keystone/config.py index 7ce9f3ca49..89ad57a960 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -5,9 +5,8 @@ from keystone import cfg class Config(cfg.ConfigOpts): def __call__(self, config_files=None, *args, **kw): - if config_files is None: - config_files = [] - self.config_file = config_files + if config_files is not None: + self.config_file = config_files super(Config, self).__call__(*args, **kw) def __getitem__(self, key, default=None): From 2c60c7f541d806f2607f29ef808cb1602abac62b Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 9 Jan 2012 15:47:09 -0800 Subject: [PATCH 149/334] adding project to keystone config to find default config files --- keystone/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/config.py b/keystone/config.py index 7ce9f3ca49..825a9ce188 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -28,7 +28,7 @@ def register_str(*args, **kw): return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) -CONF = Config() +CONF = Config(project='keystone') register_str('admin_token', default='ADMIN') From 3c88b7f546e727d8ae86901dd62b8230d1d85314 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 15:52:48 -0800 Subject: [PATCH 150/334] merge test and default configs --- keystone/test.py | 31 ++++++++++++---- tests/default.conf | 62 -------------------------------- tests/test_backend_sql.py | 4 ++- tests/test_keystoneclient.py | 11 +++--- tests/test_keystoneclient_sql.py | 4 ++- tests/test_legacy_compat.py | 5 +-- tests/test_novaclient_compat.py | 7 ++-- tests/test_overrides.conf | 2 ++ 8 files changed, 46 insertions(+), 80 deletions(-) delete mode 100644 tests/default.conf create mode 100644 tests/test_overrides.conf diff --git a/keystone/test.py b/keystone/test.py index ccd43940cd..6b100d77f3 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -19,11 +19,25 @@ from keystone import wsgi ROOTDIR = os.path.dirname(os.path.dirname(__file__)) VENDOR = os.path.join(ROOTDIR, 'vendor') TESTSDIR = os.path.join(ROOTDIR, 'tests') +ETCDIR = os.path.join(ROOTDIR, 'etc') CONF = config.CONF + cd = os.chdir +def rootdir(*p): + return os.path.join(ROOTDIR, *p) + + +def etcdir(*p): + return os.path.join(ETCDIR, *p) + + +def testsdir(*p): + return os.path.join(TESTSDIR, *p) + + def checkout_vendor(repo, rev): name = repo.split('/')[-1] if name.endswith('.git'): @@ -142,15 +156,20 @@ class TestCase(unittest.TestCase): 'metadata_%s%s' % (metadata['user_id'], metadata['tenant_id']), rv) - def loadapp(self, config, name='main'): + def _paste_config(self, config): if not config.startswith('config:'): - config = 'config:%s.conf' % os.path.join(TESTSDIR, config) - return deploy.loadapp(config, name=name) + test_path = os.path.join(TESTSDIR, config) + etc_path = os.path.join(ROOTDIR, 'etc', config) + for path in [test_path, etc_path]: + if os.path.exists('%s.conf' % path): + return 'config:%s.conf' % path + return config + + def loadapp(self, config, name='main'): + return deploy.loadapp(self._paste_config(config), name=name) def appconfig(self, config): - if not config.startswith('config:'): - config = 'config:%s.conf' % os.path.join(TESTSDIR, config) - return deploy.appconfig(config) + return deploy.appconfig(self._paste_config(config)) def serveapp(self, config, name=None): app = self.loadapp(config, name=name) diff --git a/tests/default.conf b/tests/default.conf deleted file mode 100644 index c2a716beb9..0000000000 --- a/tests/default.conf +++ /dev/null @@ -1,62 +0,0 @@ -[DEFAULT] -public_port = 5000 -admin_port = 35357 -admin_token = ADMIN -compute_port = 3000 - -[sql] -connection = sqlite:///bla.db -idle_timeout = 200 -min_pool_size = 5 -max_pool_size = 10 -pool_timeout = 200 - -[identity] -driver = keystone.backends.kvs.KvsIdentity - -[catalog] -driver = keystone.backends.templated.TemplatedCatalog -template_file = default_catalog.templates - -[token] -driver = keystone.backends.kvs.KvsToken - -[policy] -driver = keystone.backends.policy.SimpleMatch - - -[filter:debug] -paste.filter_factory = keystone.wsgi:Debug.factory - -[filter:token_auth] -paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory - -[filter:admin_token_auth] -paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory - -[filter:json_body] -paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory - -[filter:crud_extension] -paste.filter_factory = keystone.service:AdminCrudExtension.factory - - -[app:public_service] -paste.app_factory = keystone.service:public_app_factory - -[app:admin_service] -paste.app_factory = keystone.service:admin_app_factory - -[pipeline:public_api] -pipeline = token_auth admin_token_auth json_body debug public_service - -[pipeline:admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension admin_service - -[composite:main] -use = egg:Paste#urlmap -/v2.0 = public_api - -[composite:admin] -use = egg:Paste#urlmap -/v2.0 = admin_api diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index 3dac570484..24cb4a81b2 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -22,7 +22,9 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests): os.unlink('bla.db') except Exception: pass - CONF(config_files=['default.conf', 'backend_sql.conf']) + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf'), + test.testsdir('backend_sql.conf')]) sql_util.setup_test_database() self.identity_api = sql.SqlIdentity() self.load_fixtures(default_fixtures) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index fb464153be..8073a00619 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -44,14 +44,14 @@ class KcMasterTestCase(CompatTestCase): reload(ks_client) self._config() - self.public_app = self.loadapp('default', name='main') - self.admin_app = self.loadapp('default', name='admin') + self.public_app = self.loadapp('keystone', name='main') + self.admin_app = self.loadapp('keystone', name='admin') self.load_backends() self.load_fixtures(default_fixtures) - self.public_server = self.serveapp('default', name='main') - self.admin_server = self.serveapp('default', name='admin') + self.public_server = self.serveapp('keystone', name='main') + self.admin_server = self.serveapp('keystone', name='admin') # TODO(termie): is_admin is being deprecated once the policy stuff # is all working @@ -62,7 +62,8 @@ class KcMasterTestCase(CompatTestCase): dict(roles=['keystone_admin'], is_admin='1')) def _config(self): - CONF(config_files=['default.conf']) + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf')]) def foo_client(self): return self._client(username='FOO', diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py index b19aff8fb4..2adcc745d9 100644 --- a/tests/test_keystoneclient_sql.py +++ b/tests/test_keystoneclient_sql.py @@ -12,5 +12,7 @@ CONF = config.CONF class KcMasterSqlTestCase(test_keystoneclient.KcMasterTestCase): def _config(self): - CONF(config_files=['default.conf', 'backend_sql.conf']) + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf'), + test.testsdir('backend_sql.conf')]) sql_util.setup_test_database() diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index 765b9dab90..fef2b8a46f 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -117,11 +117,12 @@ class CompatTestCase(test.TestCase): class DiabloCompatTestCase(CompatTestCase): def setUp(self): - CONF(config_files=['default.conf']) + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf')]) revdir = test.checkout_vendor(KEYSTONE_REPO, 'stable/diablo') self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) - self.app = self.loadapp('default') + self.app = self.loadapp('keystone') self.load_backends() super(DiabloCompatTestCase, self).setUp() diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index d762c5f1e2..8aa6e344af 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -32,11 +32,12 @@ class NovaClientCompatMasterTestCase(CompatTestCase): reload(ks_client) reload(base_client) - CONF(config_files=['default.conf']) - self.app = self.loadapp('default') + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf')]) + self.app = self.loadapp('keystone') self.load_backends() self.load_fixtures(default_fixtures) - self.server = self.serveapp('default') + self.server = self.serveapp('keystone') def test_authenticate_and_tenants(self): from novaclient.keystone import client as ks_client diff --git a/tests/test_overrides.conf b/tests/test_overrides.conf new file mode 100644 index 0000000000..fdeb6e4b2d --- /dev/null +++ b/tests/test_overrides.conf @@ -0,0 +1,2 @@ +[catalog] +template_file = default_catalog.templates From 732909a7be4e856f2a5c2a56778a852d42eec6c0 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 17:55:33 -0800 Subject: [PATCH 151/334] add a db_sync command to bin/ks, remove others --- .gitignore | 2 +- bin/ks | 107 ++++++++++------------------------ keystone/backends/sql/core.py | 6 ++ keystone/config.py | 4 +- 4 files changed, 41 insertions(+), 78 deletions(-) diff --git a/.gitignore b/.gitignore index 4bc32265e5..3d40983549 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ run_tests.log covhtml pep8.txt nosetests.xml -tests/bla.db +bla.db docs/build .DS_Store diff --git a/bin/ks b/bin/ks index 4eaceeca3a..053d1b527e 100755 --- a/bin/ks +++ b/bin/ks @@ -1,22 +1,27 @@ #!/usr/bin/env python +import os import sys import cli.app import cli.log -from keystone import client +# If ../../keystone/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, + 'keystone', + '__init__.py')): + sys.path.insert(0, possible_topdir) -DEFAULT_PARAMS = ( - (('--config',), {'dest': 'configfile', - 'action': 'store', - 'default': './etc/default.conf'}), - (('--url',), {'dest': 'url', - 'action': 'store', - 'default': 'http://localhost:5000'}), - (('--token',), {'dest': 'token', 'action': 'store'}), - ) +from keystone import config +from keystone import utils + + +CONF = config.CONF class BaseApp(cli.log.LoggingApp): @@ -43,80 +48,32 @@ class BaseApp(cli.log.LoggingApp): return kv -class LoadData(BaseApp): - def __init__(self, *args, **kw): - super(LoadData, self).__init__(*args, **kw) - self.add_default_params() - self.add_param('fixture', nargs='+') - - def main(self): - """Given some fixtures, create the appropriate data in Keystone.""" - pass - - -class CrudCommands(BaseApp): - ACTION_MAP = {} +class DbSync(BaseApp): + name = 'db_sync' def __init__(self, *args, **kw): - super(CrudCommands, self).__init__(*args, **kw) - self.add_default_params() - self.add_param('action') - self.add_param('keyvalues', nargs='+') + super(DbSync, self).__init__(*args, **kw) def main(self): - """Given some keyvalues create the appropriate data in Keystone.""" - c = client.HttpClient(self.params.url, token=self.params.token) - action_name = self.ACTION_MAP[self.params.action] - kv = self._parse_keyvalues(self.params.keyvalues) - resp = getattr(c, action_name)(**kv) - print resp + for k in ['identity', 'catalog', 'policy', 'token']: + driver = utils.import_object(getattr(CONF, k).driver) + if hasattr(driver, 'db_sync'): + driver.db_sync() -class UserCommands(CrudCommands): - ACTION_MAP = {'add': 'create_user', - 'create': 'create_user', - } - - -class TenantCommands(CrudCommands): - ACTION_MAP = {'add': 'create_tenant', - 'create': 'create_tenant', - } - - -class ExtrasCommands(CrudCommands): - ACTION_MAP = {'add': 'create_extras', - 'create': 'create_extras', - } - - -class Auth(BaseApp): - def __init__(self, *args, **kw): - super(Auth, self).__init__(*args, **kw) - self.add_default_params() - self.add_param('keyvalues', nargs='+') - - def main(self): - """Attempt to authenticate against the Keystone API.""" - c = client.HttpClient(self.params.url, token=self.params.token) - kv = self._parse_keyvalues(self.params.keyvalues) - resp = c.authenticate(**kv) - print resp - - -CMDS = {'loaddata': LoadData, - 'user': UserCommands, - 'tenant': TenantCommands, - 'extras': ExtrasCommands, - 'auth': Auth, +CMDS = {'db_sync': DbSync, } if __name__ == '__main__': - if not len(sys.argv) > 1: - print 'try one of:', ' '.join(CMDS.keys()) - sys.exit(1) + dev_conf = os.path.join(possible_topdir, + 'etc', + 'keystone.conf') + config_files = None + if os.path.exists(dev_conf): + config_files = [dev_conf] - cmd = sys.argv[1] + args = CONF(config_files=config_files, args=sys.argv) + cmd = args[1] if cmd in CMDS: - CMDS[cmd](argv=(sys.argv[:1] + sys.argv[2:])).run() + CMDS[cmd](argv=(args[:1] + args[2:])).run() diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index a9a95bbe08..5bebf3af32 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -12,6 +12,7 @@ import sqlalchemy.engine.url from keystone import config from keystone import models +from keystone.backends.sql import migration CONF = config.CONF @@ -198,6 +199,11 @@ class SqlBase(object): class SqlIdentity(SqlBase): + # Internal interface to manage the database + def db_sync(self): + migration.db_sync() + + # Identity interface def authenticate(self, user_id=None, tenant_id=None, password=None): """Authenticate based on a user, tenant and password. diff --git a/keystone/config.py b/keystone/config.py index c2c7a9248e..796bb9b4f1 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -6,8 +6,8 @@ from keystone import cfg class Config(cfg.ConfigOpts): def __call__(self, config_files=None, *args, **kw): if config_files is not None: - self.config_file = config_files - super(Config, self).__call__(*args, **kw) + self._opts['config_file']['opt'].default = config_files + return super(Config, self).__call__(*args, **kw) def __getitem__(self, key, default=None): return getattr(self, key, default) From 6540120494b0d5ed476ed053af980547bf26e4cd Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 20:57:03 -0800 Subject: [PATCH 152/334] cli using keystoneclient --- bin/ks | 60 ++++++++++++++++++++++++++++++++++++++++++++++ keystone/config.py | 7 ++++++ 2 files changed, 67 insertions(+) diff --git a/bin/ks b/bin/ks index 053d1b527e..a07acc25e7 100755 --- a/bin/ks +++ b/bin/ks @@ -5,6 +5,7 @@ import sys import cli.app import cli.log +from keystoneclient.v2_0 import client as kc # If ../../keystone/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -22,6 +23,12 @@ from keystone import utils CONF = config.CONF +config.register_cli_str('endpoint', + default='http://localhost:$admin_port/v2.0', + group='ks') +config.register_cli_str('token', + default='$admin_token', + group='ks') class BaseApp(cli.log.LoggingApp): @@ -61,7 +68,60 @@ class DbSync(BaseApp): driver.db_sync() +class ClientCommand(BaseApp): + ACTION_MAP = None + + def __init__(self, *args, **kw): + super(ClientCommand, self).__init__(*args, **kw) + if not self.ACTION_MAP: + self.ACTION_MAP = {} + self.add_param('action') + self.add_param('keyvalues', nargs='*') + self.client = kc.Client(CONF.ks.endpoint, token=CONF.ks.token) + self.handle = getattr(self.client, '%ss' % self.__class__.__name__.lower()) + self._build_action_map() + + def _build_action_map(self): + actions = {} + for k in dir(self.handle): + if not k.startswith('_'): + actions[k] = k + self.ACTION_MAP.update(actions) + + def main(self): + """Given some keyvalues create the appropriate data in Keystone.""" + action_name = self.ACTION_MAP[self.params.action] + kv = self._parse_keyvalues(self.params.keyvalues) + resp = getattr(self.handle, action_name)(**kv) + print resp + + +class Role(ClientCommand): + pass + + +class Service(ClientCommand): + pass + + +class Token(ClientCommand): + pass + + +class Tenant(ClientCommand): + pass + + +class User(ClientCommand): + pass + + CMDS = {'db_sync': DbSync, + 'role': Role, + 'service': Service, + 'token': Token, + 'tenant': Tenant, + 'user': User, } diff --git a/keystone/config.py b/keystone/config.py index 796bb9b4f1..80c73cb8ba 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -27,6 +27,13 @@ def register_str(*args, **kw): return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) +def register_cli_str(*args, **kw): + group = kw.pop('group', None) + if group: + CONF.register_group(cfg.OptGroup(name=group)) + return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) + + CONF = Config(project='keystone') From 393aedb7ace6843b637aca7d156ffb9a430e671f Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 13:42:03 -0800 Subject: [PATCH 153/334] adding logging from configuration files, default logging per common --- bin/keystone | 9 +++- etc/keystone.conf | 11 +++++ etc/logging.conf.sample | 54 ++++++++++++++++++++++++ keystone/cfg.py | 4 ++ keystone/config.py | 92 +++++++++++++++++++++++++++++++---------- keystone/logging.py | 3 ++ tools/pip-requires | 2 + tools/pip-requires-test | 2 + 8 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 etc/logging.conf.sample diff --git a/bin/keystone b/bin/keystone index 881e517574..6e0f43982a 100755 --- a/bin/keystone +++ b/bin/keystone @@ -32,6 +32,8 @@ def create_server(conf, name, port): def serve(*servers): for server in servers: + logging.debug("starting server %s on port %s", server.application, + server.port) server.start() for server in servers: @@ -51,8 +53,11 @@ if __name__ == '__main__': CONF(config_files=config_files) - # TODO(termie): configure this based on config - logging.getLogger().setLevel(logging.DEBUG) + config.setup_logging(CONF) + + # Log the options used when starting if we're in debug mode... + if CONF.debug: + CONF.log_opt_values(logging.getLogger(CONF.prog), logging.DEBUG) options = deploy.appconfig('config:%s' % CONF.config_file[0]) diff --git a/etc/keystone.conf b/etc/keystone.conf index a1837095fb..70559af887 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -3,6 +3,17 @@ public_port = 5000 admin_port = 35357 admin_token = ADMIN compute_port = 3000 +verbose = True +debug = False +#log_config = /etc/keystone/logging.conf + +# ================= Syslog Options ============================ +# Send logs to syslog (/dev/log) instead of to file specified +# by `log-file` +use_syslog = False + +# Facility to use. If unset defaults to LOG_USER. +# syslog_log_facility = LOG_LOCAL0 [sql] connection = sqlite:///bla.db diff --git a/etc/logging.conf.sample b/etc/logging.conf.sample new file mode 100644 index 0000000000..6e3d1a042b --- /dev/null +++ b/etc/logging.conf.sample @@ -0,0 +1,54 @@ +[loggers] +keys=root,api,registry,combined + +[formatters] +keys=normal,normal_with_name,debug + +[handlers] +keys=production,file,devel + +[logger_root] +level=NOTSET +handlers=devel + +[logger_api] +level=DEBUG +handlers=devel +qualname=glance-api + +[logger_registry] +level=DEBUG +handlers=devel +qualname=glance-registry + +[logger_combined] +level=DEBUG +handlers=devel +qualname=glance-combined + +[handler_production] +class=handlers.SysLogHandler +level=ERROR +formatter=normal_with_name +args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=normal_with_name +args=('glance.log', 'w') + +[handler_devel] +class=StreamHandler +level=NOTSET +formatter=debug +args=(sys.stdout,) + +[formatter_normal] +format=%(asctime)s %(levelname)s %(message)s + +[formatter_normal_with_name] +format=(%(name)s): %(asctime)s %(levelname)s %(message)s + +[formatter_debug] +format=(%(name)s): %(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s diff --git a/keystone/cfg.py b/keystone/cfg.py index 54940e7d90..fd79e1222d 100644 --- a/keystone/cfg.py +++ b/keystone/cfg.py @@ -1115,6 +1115,10 @@ class CommonConfigOpts(ConfigOpts): StrOpt('log-dir', help='(Optional) The directory to keep log files in ' '(will be prepended to --logfile)'), + StrOpt('syslog-log-facility', + default='LOG_USER', + help='(Optional) The syslog facility to use when logging ' + 'to syslog (defaults to LOG_USER)'), BoolOpt('use-syslog', default=False, help='Use syslog for logging.'), diff --git a/keystone/config.py b/keystone/config.py index 80c73cb8ba..5e6f909cc1 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -1,42 +1,90 @@ - +import logging +import sys +import os from keystone import cfg -class Config(cfg.ConfigOpts): - def __call__(self, config_files=None, *args, **kw): - if config_files is not None: - self._opts['config_file']['opt'].default = config_files - return super(Config, self).__call__(*args, **kw) +class Config(cfg.CommonConfigOpts): + def __call__(self, config_files=None, *args, **kw): + if config_files is not None: + self._opts['config_file']['opt'].default = config_files + return super(Config, self).__call__(*args, **kw) - def __getitem__(self, key, default=None): - return getattr(self, key, default) + def __getitem__(self, key, default=None): + return getattr(self, key, default) - def __setitem__(self, key, value): - return setattr(self, key, value) + def __setitem__(self, key, value): + return setattr(self, key, value) - def iteritems(self): - for k in self._opts: - yield (k, getattr(self, k)) + def iteritems(self): + for k in self._opts: + yield (k, getattr(self, k)) def register_str(*args, **kw): - group = kw.pop('group', None) - if group: - CONF.register_group(cfg.OptGroup(name=group)) - return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + group = kw.pop('group', None) + if group: + CONF.register_group(cfg.OptGroup(name=group)) + return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) def register_cli_str(*args, **kw): - group = kw.pop('group', None) - if group: - CONF.register_group(cfg.OptGroup(name=group)) - return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) + group = kw.pop('group', None) + if group: + CONF.register_group(cfg.OptGroup(name=group)) + return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) + + +def setup_logging(conf): + """ + Sets up the logging options for a log with supplied name + + :param conf: a cfg.ConfOpts object + """ + + if conf.log_config: + # Use a logging configuration file for all settings... + if os.path.exists(conf.log_config): + logging.config.fileConfig(conf.log_config) + return + else: + raise RuntimeError("Unable to locate specified logging " + "config file: %s" % conf.log_config) + + root_logger = logging.root + if conf.debug: + root_logger.setLevel(logging.DEBUG) + elif conf.verbose: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + + formatter = logging.Formatter(conf.log_format, conf.log_date_format) + + if conf.use_syslog: + try: + facility = getattr(logging.handlers.SysLogHandler, + conf.syslog_log_facility) + except AttributeError: + raise ValueError(_("Invalid syslog facility")) + + handler = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + elif conf.log_file: + logfile = conf.log_file + if conf.log_dir: + logfile = os.path.join(conf.log_dir, logfile) + handler = logging.handlers.WatchedFileHandler(logfile) + else: + handler = logging.StreamHandler(sys.stdout) + + handler.setFormatter(formatter) + root_logger.addHandler(handler) CONF = Config(project='keystone') - register_str('admin_token', default='ADMIN') register_str('compute_port') register_str('admin_port') diff --git a/keystone/logging.py b/keystone/logging.py index c1543961e2..d7cc675606 100644 --- a/keystone/logging.py +++ b/keystone/logging.py @@ -28,6 +28,9 @@ exception = logging.exception critical = logging.critical log = logging.log +# classes +root = logging.root +Formatter = logging.Formatter # handlers StreamHandler = logging.StreamHandler diff --git a/tools/pip-requires b/tools/pip-requires index 80b03695b0..3352ed6645 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -5,3 +5,5 @@ PasteDeploy paste routes pycli +sqlalchemy +sqlalchemy-migrate diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 94adcc0570..563eaa2cbc 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -5,6 +5,8 @@ eventlet==0.9.12 PasteDeploy paste routes +sqlalchemy +sqlalchemy-migrate # keystonelight testing dependencies nose From c3c05cbabd1f88fcef68ecb834f090f82955fd45 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 14:22:41 -0800 Subject: [PATCH 154/334] adding gettext --- keystone/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/keystone/config.py b/keystone/config.py index 5e6f909cc1..5bb084c2ff 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -1,3 +1,4 @@ +import gettext import logging import sys import os @@ -83,6 +84,8 @@ def setup_logging(conf): root_logger.addHandler(handler) +gettext.install('keystone', unicode=1) + CONF = Config(project='keystone') register_str('admin_token', default='ADMIN') From d940dc40e6d8cdd688314203e9f12f2149380c19 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 14:33:15 -0800 Subject: [PATCH 155/334] fixing imports for syslog handlers and gettext --- keystone/config.py | 6 +++--- keystone/logging.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/keystone/config.py b/keystone/config.py index 5bb084c2ff..6e8e163f65 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -65,13 +65,13 @@ def setup_logging(conf): if conf.use_syslog: try: - facility = getattr(logging.handlers.SysLogHandler, + facility = getattr(logging.SysLogHandler, conf.syslog_log_facility) except AttributeError: raise ValueError(_("Invalid syslog facility")) - handler = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) + handler = logging.SysLogHandler(address='/dev/log', + facility=facility) elif conf.log_file: logfile = conf.log_file if conf.log_dir: diff --git a/keystone/logging.py b/keystone/logging.py index d7cc675606..bac4e238bf 100644 --- a/keystone/logging.py +++ b/keystone/logging.py @@ -4,6 +4,7 @@ import functools import logging import pprint +from logging.handlers import SysLogHandler # A list of things we want to replicate from logging. # levels @@ -36,7 +37,7 @@ Formatter = logging.Formatter StreamHandler = logging.StreamHandler #WatchedFileHandler = logging.handlers.WatchedFileHandler # logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. -#SysLogHandler = logging.handlers.SysLogHandler +SysLogHandler = SysLogHandler def log_debug(f): From 59614300798793e40619ce2d272bc8a7d8fd62d7 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 21:20:44 -0800 Subject: [PATCH 156/334] rename ks to keystone-manage --- bin/{ks => keystone-manage} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bin/{ks => keystone-manage} (100%) diff --git a/bin/ks b/bin/keystone-manage similarity index 100% rename from bin/ks rename to bin/keystone-manage From 230a00386014c06e01b9d79a641640f4d6d6e764 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 9 Jan 2012 21:30:09 -0800 Subject: [PATCH 157/334] add id-only flag to return IDs --- bin/keystone-manage | 6 ++++++ keystone/config.py | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index a07acc25e7..310b5bfadd 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -29,6 +29,9 @@ config.register_cli_str('endpoint', config.register_cli_str('token', default='$admin_token', group='ks') +config.register_cli_bool('id-only', + default=False, + group='ks') class BaseApp(cli.log.LoggingApp): @@ -93,6 +96,9 @@ class ClientCommand(BaseApp): action_name = self.ACTION_MAP[self.params.action] kv = self._parse_keyvalues(self.params.keyvalues) resp = getattr(self.handle, action_name)(**kv) + if CONF.ks.id_only and getattr(resp, 'id'): + print resp.id + return print resp diff --git a/keystone/config.py b/keystone/config.py index 6e8e163f65..69a4fa8e8a 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -6,6 +6,9 @@ import os from keystone import cfg +gettext.install('keystone', unicode=1) + + class Config(cfg.CommonConfigOpts): def __call__(self, config_files=None, *args, **kw): if config_files is not None: @@ -23,20 +26,6 @@ class Config(cfg.CommonConfigOpts): yield (k, getattr(self, k)) -def register_str(*args, **kw): - group = kw.pop('group', None) - if group: - CONF.register_group(cfg.OptGroup(name=group)) - return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) - - -def register_cli_str(*args, **kw): - group = kw.pop('group', None) - if group: - CONF.register_group(cfg.OptGroup(name=group)) - return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) - - def setup_logging(conf): """ Sets up the logging options for a log with supplied name @@ -84,15 +73,42 @@ def setup_logging(conf): root_logger.addHandler(handler) -gettext.install('keystone', unicode=1) +def register_str(*args, **kw): + group = _ensure_group(kw) + return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + + +def register_cli_str(*args, **kw): + group = _ensure_group(kw) + return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + + +def register_bool(*args, **kw): + group = _ensure_group(kw) + return CONF.register_opt(cfg.BoolOpt(*args, **kw), group=group) + + +def register_cli_bool(*args, **kw): + group = _ensure_group(kw) + return CONF.register_cli_opt(cfg.BoolOpt(*args, **kw), group=group) + + +def _ensure_group(kw): + group = kw.pop('group', None) + if group: + CONF.register_group(cfg.OptGroup(name=group)) + return group + CONF = Config(project='keystone') + register_str('admin_token', default='ADMIN') register_str('compute_port') register_str('admin_port') register_str('public_port') + # sql options register_str('connection', group='sql') register_str('idle_timeout', group='sql') @@ -100,6 +116,7 @@ register_str('min_pool_size', group='sql') register_str('maz_pool_size', group='sql') register_str('pool_timeout', group='sql') + register_str('driver', group='catalog') register_str('driver', group='identity') register_str('driver', group='policy') From d230857c3aa52a35f509e9c00884f59606d69fed Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 16:36:08 -0800 Subject: [PATCH 158/334] adding #vim to file with changed indent --- keystone/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keystone/config.py b/keystone/config.py index 69a4fa8e8a..e9e64a625b 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -1,3 +1,4 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 import gettext import logging import sys From ec8574928c929260b74c7a72aa8db9f66e103c67 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 16:45:39 -0800 Subject: [PATCH 159/334] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ef033b0fda..c8a3d75019 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='keystone', author_email='openstack@lists.launchpad.net', url='http://www.openstack.org', packages=find_packages(exclude=['test', 'bin']), - scripts=['bin/keystone', 'bin/ks'], + scripts=['bin/keystone', 'bin/keystone-manage'], zip_safe=False, install_requires=['setuptools'], ) From 47908a4735d757d010aa30dcab4a2d4eb410aae6 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:21:28 -0800 Subject: [PATCH 160/334] add legacy middleware --- keystone/bufferedhttp.py | 165 ++++++++ keystone/middleware/__init__.py | 1 + keystone/middleware/auth_token.py | 397 ++++++++++++++++++ .../{middleware.py => middleware/internal.py} | 0 4 files changed, 563 insertions(+) create mode 100644 keystone/bufferedhttp.py create mode 100644 keystone/middleware/__init__.py create mode 100755 keystone/middleware/auth_token.py rename keystone/{middleware.py => middleware/internal.py} (100%) diff --git a/keystone/bufferedhttp.py b/keystone/bufferedhttp.py new file mode 100644 index 0000000000..fdb35ee657 --- /dev/null +++ b/keystone/bufferedhttp.py @@ -0,0 +1,165 @@ +# Copyright (c) 2010-2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Monkey Patch httplib.HTTPResponse to buffer reads of headers. This can improve +performance when making large numbers of small HTTP requests. This module +also provides helper functions to make HTTP connections using +BufferedHTTPResponse. + +.. warning:: + + If you use this, be sure that the libraries you are using do not access + the socket directly (xmlrpclib, I'm looking at you :/), and instead + make all calls through httplib. +""" + +from urllib import quote +import logging +import time + +from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \ + HTTPResponse, HTTPSConnection, _UNKNOWN + + +class BufferedHTTPResponse(HTTPResponse): + """HTTPResponse class that buffers reading of headers""" + + def __init__(self, sock, debuglevel=0, strict=0, + method=None): # pragma: no cover + self.sock = sock + self.fp = sock.makefile('rb') + self.debuglevel = debuglevel + self.strict = strict + self._method = method + + self.msg = None + + # from the Status-Line of the response + self.version = _UNKNOWN # HTTP-Version + self.status = _UNKNOWN # Status-Code + self.reason = _UNKNOWN # Reason-Phrase + + self.chunked = _UNKNOWN # is "chunked" being used? + self.chunk_left = _UNKNOWN # bytes left to read in current chunk + self.length = _UNKNOWN # number of bytes left in response + self.will_close = _UNKNOWN # conn will close at end of response + + def expect_response(self): + self.fp = self.sock.makefile('rb', 0) + version, status, reason = self._read_status() + if status != CONTINUE: + self._read_status = lambda: (version, status, reason) + self.begin() + else: + self.status = status + self.reason = reason.strip() + self.version = 11 + self.msg = HTTPMessage(self.fp, 0) + self.msg.fp = None + + +class BufferedHTTPConnection(HTTPConnection): + """HTTPConnection class that uses BufferedHTTPResponse""" + response_class = BufferedHTTPResponse + + def connect(self): + self._connected_time = time.time() + return HTTPConnection.connect(self) + + def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): + self._method = method + self._path = url + return HTTPConnection.putrequest(self, method, url, skip_host, + skip_accept_encoding) + + def getexpect(self): + response = BufferedHTTPResponse(self.sock, strict=self.strict, + method=self._method) + response.expect_response() + return response + + def getresponse(self): + response = HTTPConnection.getresponse(self) + logging.debug(("HTTP PERF: %(time).5f seconds to %(method)s " + "%(host)s:%(port)s %(path)s)"), + {'time': time.time() - self._connected_time, 'method': self._method, + 'host': self.host, 'port': self.port, 'path': self._path}) + return response + + +def http_connect(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + """ + Helper function to create an HTTPConnection object. If ssl is set True, + HTTPSConnection will be used. However, if ssl=False, BufferedHTTPConnection + will be used, which is buffered for backend Swift services. + + :param ipaddr: IPv4 address to connect to + :param port: port to connect to + :param device: device of the node to query + :param partition: partition on the device + :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) + :param path: request path + :param headers: dictionary of headers + :param query_string: request query string + :param ssl: set True if SSL should be used (default: False) + :returns: HTTPConnection object + """ + if ssl: + conn = HTTPSConnection('%s:%s' % (ipaddr, port)) + else: + conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) + path = quote('/' + device + '/' + str(partition) + path) + if query_string: + path += '?' + query_string + conn.path = path + conn.putrequest(method, path) + if headers: + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() + return conn + + +def http_connect_raw(ipaddr, port, method, path, headers=None, + query_string=None, ssl=False): + """ + Helper function to create an HTTPConnection object. If ssl is set True, + HTTPSConnection will be used. However, if ssl=False, BufferedHTTPConnection + will be used, which is buffered for backend Swift services. + + :param ipaddr: IPv4 address to connect to + :param port: port to connect to + :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) + :param path: request path + :param headers: dictionary of headers + :param query_string: request query string + :param ssl: set True if SSL should be used (default: False) + :returns: HTTPConnection object + """ + if ssl: + conn = HTTPSConnection('%s:%s' % (ipaddr, port)) + else: + conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) + if query_string: + path += '?' + query_string + conn.path = path + conn.putrequest(method, path) + if headers: + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() + return conn diff --git a/keystone/middleware/__init__.py b/keystone/middleware/__init__.py new file mode 100644 index 0000000000..1593e6e2f4 --- /dev/null +++ b/keystone/middleware/__init__.py @@ -0,0 +1 @@ +from keystone.middleware.internal import * diff --git a/keystone/middleware/auth_token.py b/keystone/middleware/auth_token.py new file mode 100755 index 0000000000..c4e28589d0 --- /dev/null +++ b/keystone/middleware/auth_token.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2010-2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +TOKEN-BASED AUTH MIDDLEWARE + +This WSGI component performs multiple jobs: + +* it verifies that incoming client requests have valid tokens by verifying + tokens with the auth service. +* it will reject unauthenticated requests UNLESS it is in 'delay_auth_decision' + mode, which means the final decision is delegated to the downstream WSGI + component (usually the OpenStack service) +* it will collect and forward identity information from a valid token + such as user name etc... + +Refer to: http://wiki.openstack.org/openstack-authn + + +HEADERS +------- + +* Headers starting with HTTP\_ is a standard http header +* Headers starting with HTTP_X is an extended http header + +Coming in from initial call from client or customer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HTTP_X_AUTH_TOKEN + the client token being passed in + +HTTP_X_STORAGE_TOKEN + the client token being passed in (legacy Rackspace use) to support + cloud files + +Used for communication between components +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +www-authenticate + only used if this component is being used remotely + +HTTP_AUTHORIZATION + basic auth password used to validate the connection + +What we add to the request for use by the OpenStack service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HTTP_X_AUTHORIZATION + the client identity being passed in + +""" +import httplib +import json +import os + +import eventlet +from eventlet import wsgi +from paste import deploy +from urlparse import urlparse +import webob +import webob.exc +from webob.exc import HTTPUnauthorized + +from keystone.bufferedhttp import http_connect_raw as http_connect + +PROTOCOL_NAME = "Token Authentication" + + +class AuthProtocol(object): + """Auth Middleware that handles authenticating client calls""" + + def _init_protocol_common(self, app, conf): + """ Common initialization code""" + print "Starting the %s component" % PROTOCOL_NAME + + self.conf = conf + self.app = app + #if app is set, then we are in a WSGI pipeline and requests get passed + # on to app. If it is not set, this component should forward requests + + # where to find the OpenStack service (if not in local WSGI chain) + # these settings are only used if this component is acting as a proxy + # and the OpenSTack service is running remotely + self.service_protocol = conf.get('service_protocol', 'https') + self.service_host = conf.get('service_host') + self.service_port = int(conf.get('service_port')) + self.service_url = '%s://%s:%s' % (self.service_protocol, + self.service_host, + self.service_port) + # used to verify this component with the OpenStack service or PAPIAuth + self.service_pass = conf.get('service_pass') + + # delay_auth_decision means we still allow unauthenticated requests + # through and we let the downstream service make the final decision + self.delay_auth_decision = int(conf.get('delay_auth_decision', 0)) + + def _init_protocol(self, conf): + """ Protocol specific initialization """ + + # where to find the auth service (we use this to validate tokens) + self.auth_host = conf.get('auth_host') + self.auth_port = int(conf.get('auth_port')) + self.auth_protocol = conf.get('auth_protocol', 'https') + + # where to tell clients to find the auth service (default to url + # constructed based on endpoint we have for the service to use) + self.auth_location = conf.get('auth_uri', + "%s://%s:%s" % (self.auth_protocol, + self.auth_host, + self.auth_port)) + + # Credentials used to verify this component with the Auth service since + # validating tokens is a privileged call + self.admin_token = conf.get('admin_token') + + def __init__(self, app, conf): + """ Common initialization code """ + + #TODO(ziad): maybe we refactor this into a superclass + self._init_protocol_common(app, conf) # Applies to all protocols + self._init_protocol(conf) # Specific to this protocol + + def __call__(self, env, start_response): + """ Handle incoming request. Authenticate. And send downstream. """ + + #Prep headers to forward request to local or remote downstream service + proxy_headers = env.copy() + for header in proxy_headers.iterkeys(): + if header[0:5] == 'HTTP_': + proxy_headers[header[5:]] = proxy_headers[header] + del proxy_headers[header] + + #Look for authentication claims + claims = self._get_claims(env) + if not claims: + #No claim(s) provided + if self.delay_auth_decision: + #Configured to allow downstream service to make final decision. + #So mark status as Invalid and forward the request downstream + self._decorate_request("X_IDENTITY_STATUS", + "Invalid", env, proxy_headers) + else: + #Respond to client as appropriate for this auth protocol + return self._reject_request(env, start_response) + else: + # this request is presenting claims. Let's validate them + valid = self._validate_claims(claims) + if not valid: + # Keystone rejected claim + if self.delay_auth_decision: + # Downstream service will receive call still and decide + self._decorate_request("X_IDENTITY_STATUS", + "Invalid", env, proxy_headers) + else: + #Respond to client as appropriate for this auth protocol + return self._reject_claims(env, start_response) + else: + self._decorate_request("X_IDENTITY_STATUS", + "Confirmed", env, proxy_headers) + + #Collect information about valid claims + if valid: + claims = self._expound_claims(claims) + + # Store authentication data + if claims: + self._decorate_request('X_AUTHORIZATION', "Proxy %s" % + claims['user'], env, proxy_headers) + + # For legacy compatibility before we had ID and Name + self._decorate_request('X_TENANT', + claims['tenant'], env, proxy_headers) + + # Services should use these + self._decorate_request('X_TENANT_NAME', + claims.get('tenant_name', claims['tenant']), + env, proxy_headers) + self._decorate_request('X_TENANT_ID', + claims['tenant'], env, proxy_headers) + + self._decorate_request('X_USER', + claims['user'], env, proxy_headers) + if 'roles' in claims and len(claims['roles']) > 0: + if claims['roles'] != None: + roles = '' + for role in claims['roles']: + if len(roles) > 0: + roles += ',' + roles += role + self._decorate_request('X_ROLE', + roles, env, proxy_headers) + + # NOTE(todd): unused + self.expanded = True + + #Send request downstream + return self._forward_request(env, start_response, proxy_headers) + + # NOTE(todd): unused + def get_admin_auth_token(self, username, password): + """ + This function gets an admin auth token to be used by this service to + validate a user's token. Validate_token is a priviledged call so + it needs to be authenticated by a service that is calling it + """ + headers = {"Content-type": "application/json", + "Accept": "application/json"} + params = {"passwordCredentials": {"username": username, + "password": password, + "tenantId": "1"}} + conn = httplib.HTTPConnection("%s:%s" \ + % (self.auth_host, self.auth_port)) + conn.request("POST", "/v2.0/tokens", json.dumps(params), \ + headers=headers) + response = conn.getresponse() + data = response.read() + return data + + def _get_claims(self, env): + """Get claims from request""" + claims = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) + return claims + + def _reject_request(self, env, start_response): + """Redirect client to auth server""" + return webob.exc.HTTPUnauthorized("Authentication required", + [("WWW-Authenticate", + "Keystone uri='%s'" % self.auth_location)])(env, + start_response) + + def _reject_claims(self, env, start_response): + """Client sent bad claims""" + return webob.exc.HTTPUnauthorized()(env, + start_response) + + def _validate_claims(self, claims): + """Validate claims, and provide identity information isf applicable """ + + # Step 1: We need to auth with the keystone service, so get an + # admin token + #TODO(ziad): Need to properly implement this, where to store creds + # for now using token from ini + #auth = self.get_admin_auth_token("admin", "secrete", "1") + #admin_token = json.loads(auth)["auth"]["token"]["id"] + + # Step 2: validate the user's token with the auth service + # since this is a priviledged op,m we need to auth ourselves + # by using an admin token + headers = {"Content-type": "application/json", + "Accept": "application/json", + "X-Auth-Token": self.admin_token} + ##TODO(ziad):we need to figure out how to auth to keystone + #since validate_token is a priviledged call + #Khaled's version uses creds to get a token + # "X-Auth-Token": admin_token} + # we're using a test token from the ini file for now + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/v2.0/tokens/%s' % claims, headers=headers) + resp = conn.getresponse() + # data = resp.read() + conn.close() + + if not str(resp.status).startswith('20'): + # Keystone rejected claim + return False + else: + #TODO(Ziad): there is an optimization we can do here. We have just + #received data from Keystone that we can use instead of making + #another call in _expound_claims + return True + + def _expound_claims(self, claims): + # Valid token. Get user data and put it in to the call + # so the downstream service can use it + headers = {"Content-type": "application/json", + "Accept": "application/json", + "X-Auth-Token": self.admin_token} + ##TODO(ziad):we need to figure out how to auth to keystone + #since validate_token is a priviledged call + #Khaled's version uses creds to get a token + # "X-Auth-Token": admin_token} + # we're using a test token from the ini file for now + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/v2.0/tokens/%s' % claims, headers=headers) + resp = conn.getresponse() + data = resp.read() + conn.close() + + if not str(resp.status).startswith('20'): + raise LookupError('Unable to locate claims: %s' % resp.status) + + token_info = json.loads(data) + roles = [] + role_refs = token_info["access"]["user"]["roles"] + if role_refs != None: + for role_ref in role_refs: + # Nova looks for the non case-sensitive role 'Admin' + # to determine admin-ness + roles.append(role_ref["name"]) + + try: + tenant = token_info['access']['token']['tenant']['id'] + tenant_name = token_info['access']['token']['tenant']['name'] + except: + tenant = None + tenant_name = None + if not tenant: + tenant = token_info['access']['user'].get('tenantId') + tenant_name = token_info['access']['user'].get('tenantName') + verified_claims = {'user': token_info['access']['user']['username'], + 'tenant': tenant, + 'roles': roles} + if tenant_name: + verified_claims['tenantName'] = tenant_name + return verified_claims + + def _decorate_request(self, index, value, env, proxy_headers): + """Add headers to request""" + proxy_headers[index] = value + env["HTTP_%s" % index] = value + + def _forward_request(self, env, start_response, proxy_headers): + """Token/Auth processed & claims added to headers""" + self._decorate_request('AUTHORIZATION', + "Basic %s" % self.service_pass, env, proxy_headers) + #now decide how to pass on the call + if self.app: + # Pass to downstream WSGI component + return self.app(env, start_response) + #.custom_start_response) + else: + # We are forwarding to a remote service (no downstream WSGI app) + req = webob.Request(proxy_headers) + parsed = urlparse(req.url) + + conn = http_connect(self.service_host, + self.service_port, + req.method, + parsed.path, + proxy_headers, + ssl=(self.service_protocol == 'https')) + resp = conn.getresponse() + data = resp.read() + + #TODO(ziad): use a more sophisticated proxy + # we are rewriting the headers now + + if resp.status == 401 or resp.status == 305: + # Add our own headers to the list + headers = [("WWW_AUTHENTICATE", + "Keystone uri='%s'" % self.auth_location)] + return webob.Response(status=resp.status, + body=data, + headerlist=headers)(env, start_response) + else: + return webob.Response(status=resp.status, + body=data)(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return AuthProtocol(app, conf) + return auth_filter + + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return AuthProtocol(None, conf) + +if __name__ == "__main__": + app = deploy.loadapp("config:" + \ + os.path.join(os.path.abspath(os.path.dirname(__file__)), + os.pardir, + os.pardir, + "examples/paste/auth_token.ini"), + global_conf={"log_name": "auth_token.log"}) + wsgi.server(eventlet.listen(('', 8090)), app) diff --git a/keystone/middleware.py b/keystone/middleware/internal.py similarity index 100% rename from keystone/middleware.py rename to keystone/middleware/internal.py From 52da8917d157ffacd05aa8dee2af5448c40766e9 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:27:04 -0800 Subject: [PATCH 161/334] add glance middleware ?? --- keystone/glance_auth_token.py | 78 +++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 keystone/glance_auth_token.py diff --git a/keystone/glance_auth_token.py b/keystone/glance_auth_token.py new file mode 100644 index 0000000000..6bef13901c --- /dev/null +++ b/keystone/glance_auth_token.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Glance Keystone Integration Middleware + +This WSGI component allows keystone to act as an identity service for +glance. Glance now supports the concept of images owned by a tenant, +and this middleware takes the authentication information provided by +auth_token and builds a glance-compatible context object. + +Use by applying after auth_token in the glance-api.ini and +glance-registry.ini configurations, replacing the existing context +middleware. + +Example: examples/paste/glance-api.conf, + examples/paste/glance-registry.conf +""" + +from glance.common import context + + +class KeystoneContextMiddleware(context.ContextMiddleware): + """Glance keystone integration middleware.""" + + def process_request(self, req): + """ + Extract keystone-provided authentication information from the + request and construct an appropriate context from it. + """ + # Only accept the authentication information if the identity + # has been confirmed--presumably by upstream + if req.headers.get('X_IDENTITY_STATUS', 'Invalid') != 'Confirmed': + # Use the default empty context + req.context = self.make_context(read_only=True) + return + + # OK, let's extract the information we need + auth_tok = req.headers.get('X_AUTH_TOKEN', + req.headers.get('X_STORAGE_TOKEN')) + user = req.headers.get('X_USER') + tenant = req.headers.get('X_TENANT') + roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] + is_admin = 'Admin' in roles + + # Construct the context + req.context = self.make_context(auth_tok=auth_tok, + user=user, + tenant=tenant, + roles=roles, + is_admin=is_admin) + + +def filter_factory(global_conf, **local_conf): + """ + Factory method for paste.deploy + """ + conf = global_conf.copy() + conf.update(local_conf) + + def filter(app): + return KeystoneContextMiddleware(app, conf) + + return filter From cd37b051e6334bfb82853dde80cc94644fc0d99b Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:29:37 -0800 Subject: [PATCH 162/334] woops --- keystone/{ => middleware}/glance_auth_token.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename keystone/{ => middleware}/glance_auth_token.py (100%) diff --git a/keystone/glance_auth_token.py b/keystone/middleware/glance_auth_token.py similarity index 100% rename from keystone/glance_auth_token.py rename to keystone/middleware/glance_auth_token.py From 2723439149de2e340edafa77719d29d4e10acf79 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:36:22 -0800 Subject: [PATCH 163/334] add a noop controller --- keystone/service.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/keystone/service.py b/keystone/service.py index 190774f2bc..8fbe18bc4b 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -141,6 +141,11 @@ class PublicRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() + noop_controller = NoopController() + mapper.connect('/', + controller=noop_controller, + action='noop') + # Token Operations auth_controller = TokenController() mapper.connect('/tokens', @@ -325,6 +330,14 @@ class AdminCrudExtension(wsgi.ExtensionRouter): application, mapper) +class NoopController(Application): + def __init__(self): + super(NoopController, self).__init__() + + def noop(self, context): + return {} + + class TokenController(Application): def __init__(self): self.catalog_api = catalog.Manager() From c8303054343a6693387c56417364495521598568 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 17:38:15 -0800 Subject: [PATCH 164/334] logging to debugging by default for now --- etc/keystone.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/keystone.conf b/etc/keystone.conf index 70559af887..517cfd43a5 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -4,7 +4,7 @@ admin_port = 35357 admin_token = ADMIN compute_port = 3000 verbose = True -debug = False +debug = True #log_config = /etc/keystone/logging.conf # ================= Syslog Options ============================ From ef1a4746045e4cac814ee46d369b367e37b58243 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 10 Jan 2012 17:44:03 -0800 Subject: [PATCH 165/334] fixing WatchedFileHandler --- keystone/config.py | 2 +- keystone/logging.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/keystone/config.py b/keystone/config.py index e9e64a625b..c99a6aff19 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -66,7 +66,7 @@ def setup_logging(conf): logfile = conf.log_file if conf.log_dir: logfile = os.path.join(conf.log_dir, logfile) - handler = logging.handlers.WatchedFileHandler(logfile) + handler = logging.WatchedFileHandler(logfile) else: handler = logging.StreamHandler(sys.stdout) diff --git a/keystone/logging.py b/keystone/logging.py index bac4e238bf..84e3e66d30 100644 --- a/keystone/logging.py +++ b/keystone/logging.py @@ -5,6 +5,7 @@ import logging import pprint from logging.handlers import SysLogHandler +from logging.handlers import WatchedFileHandler # A list of things we want to replicate from logging. # levels @@ -35,8 +36,7 @@ Formatter = logging.Formatter # handlers StreamHandler = logging.StreamHandler -#WatchedFileHandler = logging.handlers.WatchedFileHandler -# logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. +WatchedFileHandler = WatchedFileHandler SysLogHandler = SysLogHandler From 61ecf604910bbd99a2007dc10f69e3a772f60398 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:50:32 -0800 Subject: [PATCH 166/334] add more middleware --- keystone/middleware/ec2_token.py | 92 ++++++++++ keystone/middleware/nova_auth_token.py | 105 +++++++++++ keystone/middleware/swift_auth.py | 243 +++++++++++++++++++++++++ 3 files changed, 440 insertions(+) create mode 100644 keystone/middleware/ec2_token.py create mode 100644 keystone/middleware/nova_auth_token.py create mode 100755 keystone/middleware/swift_auth.py diff --git a/keystone/middleware/ec2_token.py b/keystone/middleware/ec2_token.py new file mode 100644 index 0000000000..af1416273f --- /dev/null +++ b/keystone/middleware/ec2_token.py @@ -0,0 +1,92 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Starting point for routing EC2 requests. + +""" + +from urlparse import urlparse + +from eventlet.green import httplib +import webob.dec +import webob.exc + +from nova import flags +from nova import utils +from nova import wsgi + + +FLAGS = flags.FLAGS +flags.DEFINE_string('keystone_ec2_url', + 'http://localhost:5000/v2.0/ec2tokens', + 'URL to get token from ec2 request.') + + +class EC2Token(wsgi.Middleware): + """Authenticate an EC2 request with keystone and convert to token.""" + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + # Read request signature and access id. + try: + signature = req.params['Signature'] + access = req.params['AWSAccessKeyId'] + except KeyError: + raise webob.exc.HTTPBadRequest() + + # Make a copy of args for authentication and signature verification. + auth_params = dict(req.params) + # Not part of authentication args + auth_params.pop('Signature') + + # Authenticate the request. + creds = {'ec2Credentials': {'access': access, + 'signature': signature, + 'host': req.host, + 'verb': req.method, + 'path': req.path, + 'params': auth_params, + }} + creds_json = utils.dumps(creds) + headers = {'Content-Type': 'application/json'} + + # Disable "has no x member" pylint error + # for httplib and urlparse + # pylint: disable-msg=E1101 + o = urlparse(FLAGS.keystone_ec2_url) + if o.scheme == "http": + conn = httplib.HTTPConnection(o.netloc) + else: + conn = httplib.HTTPSConnection(o.netloc) + conn.request('POST', o.path, body=creds_json, headers=headers) + response = conn.getresponse().read() + conn.close() + + # NOTE(vish): We could save a call to keystone by + # having keystone return token, tenant, + # user, and roles from this call. + + result = utils.loads(response) + try: + token_id = result['access']['token']['id'] + except (AttributeError, KeyError): + raise webob.exc.HTTPBadRequest() + + # Authenticated! + req.headers['X-Auth-Token'] = token_id + return self.application diff --git a/keystone/middleware/nova_auth_token.py b/keystone/middleware/nova_auth_token.py new file mode 100644 index 0000000000..68ad1d9db3 --- /dev/null +++ b/keystone/middleware/nova_auth_token.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2010-2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +NOVA LAZY PROVISIONING AUTH MIDDLEWARE + +This WSGI component allows keystone act as an identity service for nova by +lazy provisioning nova projects/users as authenticated by auth_token. + +Use by applying after auth_token in the nova paste config. +Example: docs/nova-api-paste.ini +""" + +from nova import auth +from nova import context +from nova import flags +from nova import utils +from nova import wsgi +from nova import exception +import webob.dec +import webob.exc + + +FLAGS = flags.FLAGS + + +class KeystoneAuthShim(wsgi.Middleware): + """Lazy provisioning nova project/users from keystone tenant/user""" + + def __init__(self, application, db_driver=None): + if not db_driver: + db_driver = FLAGS.db_driver + self.db = utils.import_object(db_driver) + self.auth = auth.manager.AuthManager() + super(KeystoneAuthShim, self).__init__(application) + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + # find or create user + try: + user_id = req.headers['X_USER'] + except: + return webob.exc.HTTPUnauthorized() + try: + user_ref = self.auth.get_user(user_id) + except: + user_ref = self.auth.create_user(user_id) + + # get the roles + roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] + + # set user admin-ness to keystone admin-ness + # FIXME: keystone-admin-role value from keystone.conf is not + # used neither here nor in glance_auth_token! + roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] + is_admin = 'Admin' in roles + if user_ref.is_admin() != is_admin: + self.auth.modify_user(user_ref, admin=is_admin) + + # create a project for tenant + if 'X_TENANT_ID' in req.headers: + # This is the new header since Keystone went to ID/Name + project_id = req.headers['X_TENANT_ID'] + else: + # This is for legacy compatibility + project_id = req.headers['X_TENANT'] + + if project_id: + try: + project_ref = self.auth.get_project(project_id) + except: + project_ref = self.auth.create_project(project_id, user_id) + # ensure user is a member of project + if not self.auth.is_project_member(user_id, project_id): + self.auth.add_to_project(user_id, project_id) + else: + project_ref = None + + # Get the auth token + auth_token = req.headers.get('X_AUTH_TOKEN', + req.headers.get('X_STORAGE_TOKEN')) + + # Build a context, including the auth_token... + ctx = context.RequestContext(user_id, project_id, + is_admin=('Admin' in roles), + auth_token=auth_token) + + req.environ['nova.context'] = ctx + return self.application diff --git a/keystone/middleware/swift_auth.py b/keystone/middleware/swift_auth.py new file mode 100755 index 0000000000..75120c340e --- /dev/null +++ b/keystone/middleware/swift_auth.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2010-2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +TOKEN-BASED AUTH MIDDLEWARE FOR SWIFT + +Authentication on incoming request + * grab token from X-Auth-Token header + * TODO: grab the memcache servers from the request env + * TODOcheck for auth information in memcache + * check for auth information from keystone + * return if unauthorized + * decorate the request for authorization in swift + * forward to the swift proxy app + +Authorization via callback + * check the path and extract the tenant + * get the auth information stored in keystone.identity during + authentication + * TODO: check if the user is an account admin or a reseller admin + * determine what object-type to authorize (account, container, object) + * use knowledge of tenant, admin status, and container acls to authorize + +""" + +import json +from urlparse import urlparse +from webob.exc import HTTPUnauthorized, HTTPNotFound, HTTPExpectationFailed + +from keystone.bufferedhttp import http_connect_raw as http_connect + +from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed +from swift.common.utils import get_logger, split_path + + +PROTOCOL_NAME = "Swift Token Authentication" + + +class AuthProtocol(object): + """Handles authenticating and aurothrizing client calls. + + Add to your pipeline in paste config like: + + [pipeline:main] + pipeline = catch_errors healthcheck cache keystone proxy-server + + [filter:keystone] + use = egg:keystone#swiftauth + keystone_url = http://127.0.0.1:8080 + keystone_admin_token = 999888777666 + """ + + def __init__(self, app, conf): + """Store valuable bits from the conf and set up logging.""" + self.app = app + self.keystone_url = urlparse(conf.get('keystone_url')) + self.admin_token = conf.get('keystone_admin_token') + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH') + self.log = get_logger(conf, log_route='keystone') + self.log.info('Keystone middleware started') + + def __call__(self, env, start_response): + """Authenticate the incoming request. + + If authentication fails return an appropriate http status here, + otherwise forward through the rest of the app. + """ + + self.log.debug('Keystone middleware called') + token = self._get_claims(env) + self.log.debug('token: %s', token) + if token: + identity = self._validate_claims(token) + if identity: + self.log.debug('request authenticated: %r', identity) + return self.perform_authenticated_request(identity, env, + start_response) + else: + self.log.debug('anonymous request') + return self.unauthorized_request(env, start_response) + self.log.debug('no auth token in request headers') + return self.perform_unidentified_request(env, start_response) + + def unauthorized_request(self, env, start_response): + """Clinet provided a token that wasn't acceptable, error out.""" + return HTTPUnauthorized()(env, start_response) + + def unauthorized(self, req): + """Return unauthorized given a webob Request object. + + This can be stuffed into the evironment for swift.authorize or + called from the authoriztion callback when authorization fails. + """ + return HTTPUnauthorized(request=req) + + def perform_authenticated_request(self, identity, env, start_response): + """Client provieded a valid identity, so use it for authorization.""" + env['keystone.identity'] = identity + env['swift.authorize'] = self.authorize + env['swift.clean_acl'] = clean_acl + self.log.debug('calling app: %s // %r', start_response, env) + rv = self.app(env, start_response) + self.log.debug('return from app: %r', rv) + return rv + + def perform_unidentified_request(self, env, start_response): + """Withouth authentication data, use acls for access control.""" + env['swift.authorize'] = self.authorize_via_acl + env['swift.clean_acl'] = self.authorize_via_acl + return self.app(env, start_response) + + def authorize(self, req): + """Used when we have a valid identity from keystone.""" + self.log.debug('keystone middleware authorization begin') + env = req.environ + tenant = env.get('keystone.identity', {}).get('tenant') + if not tenant: + self.log.warn('identity info not present in authorize request') + return HTTPExpectationFailed('Unable to locate auth claim', + request=req) + # TODO(todd): everyone under a tenant can do anything to that tenant. + # more realistic would be role/group checking to do things + # like deleting the account or creating/deleting containers + # esp. when owned by other users in the same tenant. + if req.path.startswith('/v1/%s_%s' % (self.reseller_prefix, tenant)): + self.log.debug('AUTHORIZED OKAY') + return None + + self.log.debug('tenant mismatch: %r', tenant) + return self.unauthorized(req) + + def authorize_via_acl(self, req): + """Anon request handling. + + For now this only allows anon read of objects. Container and account + actions are prohibited. + """ + + self.log.debug('authorizing anonymous request') + try: + version, account, container, obj = split_path(req.path, 1, 4, True) + except ValueError: + return HTTPNotFound(request=req) + + if obj: + return self._authorize_anon_object(req, account, container, obj) + + if container: + return self._authorize_anon_container(req, account, container) + + if account: + return self._authorize_anon_account(req, account) + + return self._authorize_anon_toplevel(req) + + def _authorize_anon_object(self, req, account, container, obj): + referrers, groups = parse_acl(getattr(req, 'acl', None)) + if referrer_allowed(req.referer, referrers): + self.log.debug('anonymous request AUTHORIZED OKAY') + return None + return self.unauthorized(req) + + def _authorize_anon_container(self, req, account, container): + return self.unauthorized(req) + + def _authorize_anon_account(self, req, account): + return self.unauthorized(req) + + def _authorize_anon_toplevel(self, req): + return self.unauthorized(req) + + def _get_claims(self, env): + claims = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) + return claims + + def _validate_claims(self, claims): + """Ask keystone (as keystone admin) for information for this user.""" + + # TODO(todd): cache + + self.log.debug('Asking keystone to validate token') + headers = {"Content-type": "application/json", + "Accept": "application/json", + "X-Auth-Token": self.admin_token} + self.log.debug('headers: %r', headers) + self.log.debug('url: %s', self.keystone_url) + conn = http_connect(self.keystone_url.hostname, self.keystone_url.port, + 'GET', '/v2.0/tokens/%s' % claims, headers=headers) + resp = conn.getresponse() + data = resp.read() + conn.close() + + # Check http status code for the "OK" family of responses + if not str(resp.status).startswith('20'): + return False + + identity_info = json.loads(data) + roles = [] + role_refs = identity_info["access"]["user"]["roles"] + + if role_refs is not None: + for role_ref in role_refs: + roles.append(role_ref["id"]) + + try: + tenant = identity_info['access']['token']['tenantId'] + except: + tenant = None + if not tenant: + tenant = identity_info['access']['user']['tenantId'] + # TODO(Ziad): add groups back in + identity = {'user': identity_info['access']['user']['username'], + 'tenant': tenant, + 'roles': roles} + + return identity + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return AuthProtocol(app, conf) + + return auth_filter From 4ae246d68837a8df6c299fe69141c38496a8217a Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 10 Jan 2012 17:55:11 -0800 Subject: [PATCH 167/334] flush that sht --- keystone/backends/sql/core.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index 5bebf3af32..d08915d260 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -285,6 +285,7 @@ class SqlIdentity(SqlBase): session = self.get_session() with session.begin(): session.add(UserTenantMembership(user_id=user_id, tenant_id=tenant_id)) + session.flush() def remove_user_from_tenant(self, tenant_id, user_id): session = self.get_session() @@ -294,6 +295,7 @@ class SqlIdentity(SqlBase): .first() with session.begin(): session.delete(membership_ref) + session.flush() def get_tenants_for_user(self, user_id): session = self.get_session() @@ -343,6 +345,7 @@ class SqlIdentity(SqlBase): with session.begin(): user_ref = User.from_dict(user) session.add(user_ref) + session.flush() return user_ref.to_dict() def update_user(self, id, user): @@ -356,6 +359,7 @@ class SqlIdentity(SqlBase): user_ref.name = new_user.name user_ref.extra = new_user.extra + session.flush() return user_ref def delete_user(self, id): @@ -363,12 +367,14 @@ class SqlIdentity(SqlBase): user_ref = session.query(User).filter_by(id=id).first() with session.begin(): session.delete(user_ref) + session.flush() def create_tenant(self, id, tenant): session = self.get_session() with session.begin(): tenant_ref = Tenant.from_dict(tenant) session.add(tenant_ref) + session.flush() return tenant_ref.to_dict() def update_tenant(self, id, tenant): @@ -382,6 +388,7 @@ class SqlIdentity(SqlBase): tenant_ref.name = new_tenant.name tenant_ref.extra = new_tenant.extra + session.flush() return tenant_ref def delete_tenant(self, id): @@ -389,6 +396,7 @@ class SqlIdentity(SqlBase): tenant_ref = session.query(Tenant).filter_by(id=id).first() with session.begin(): session.delete(tenant_ref) + session.flush() def create_metadata(self, user_id, tenant_id, metadata): session = self.get_session() @@ -396,6 +404,7 @@ class SqlIdentity(SqlBase): session.add(Metadata(user_id=user_id, tenant_id=tenant_id, data=metadata)) + session.flush() return metadata def update_metadata(self, user_id, tenant_id, metadata): @@ -409,6 +418,7 @@ class SqlIdentity(SqlBase): for k in metadata: data[k] = metadata[k] metadata_ref.data = data + session.flush() return metadata_ref def delete_metadata(self, user_id, tenant_id): @@ -419,6 +429,7 @@ class SqlIdentity(SqlBase): session = self.get_session() with session.begin(): session.add(Role(**role)) + session.flush() return role def update_role(self, id, role): @@ -427,6 +438,7 @@ class SqlIdentity(SqlBase): role_ref = session.query(Role).filter_by(id=id).first() for k in role: role_ref[k] = role[k] + session.flush() return role_ref def delete_role(self, id): From c25155acf9a40caea38d62fddd5dd9d18b56106a Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 12:57:20 -0800 Subject: [PATCH 168/334] check for membership --- keystone/backends/sql/core.py | 7 ++ keystone/middleware/nova_keystone_context.py | 69 ++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 keystone/middleware/nova_keystone_context.py diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index d08915d260..e6862c1f03 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -283,6 +283,13 @@ class SqlIdentity(SqlBase): # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): session = self.get_session() + q = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id) + rv = q.first() + if rv: + return + with session.begin(): session.add(UserTenantMembership(user_id=user_id, tenant_id=tenant_id)) session.flush() diff --git a/keystone/middleware/nova_keystone_context.py b/keystone/middleware/nova_keystone_context.py new file mode 100644 index 0000000000..5c41bc87da --- /dev/null +++ b/keystone/middleware/nova_keystone_context.py @@ -0,0 +1,69 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Nova Auth Middleware. + +""" + +import webob.dec +import webob.exc + +from nova import context +from nova import flags +from nova import wsgi + + +FLAGS = flags.FLAGS +flags.DECLARE('use_forwarded_for', 'nova.api.auth') + + +class NovaKeystoneContext(wsgi.Middleware): + """Make a request context from keystone headers""" + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + try: + user_id = req.headers['X_USER'] + except KeyError: + return webob.exc.HTTPUnauthorized() + # get the roles + roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] + + if 'X_TENANT_ID' in req.headers: + # This is the new header since Keystone went to ID/Name + project_id = req.headers['X_TENANT_ID'] + else: + # This is for legacy compatibility + project_id = req.headers['X_TENANT'] + + # Get the auth token + auth_token = req.headers.get('X_AUTH_TOKEN', + req.headers.get('X_STORAGE_TOKEN')) + + # Build a context, including the auth_token... + remote_address = getattr(req, 'remote_address', '127.0.0.1') + remote_address = req.remote_addr + if FLAGS.use_forwarded_for: + remote_address = req.headers.get('X-Forwarded-For', remote_address) + ctx = context.RequestContext(user_id, + project_id, + roles=roles, + auth_token=auth_token, + strategy='keystone', + remote_address=remote_address) + + req.environ['nova.context'] = ctx + return self.application From aea09bd48e73a1b5dbaec88a971e681ad6e95c8c Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 13:33:44 -0800 Subject: [PATCH 169/334] fix token auth --- keystone/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/keystone/service.py b/keystone/service.py index 8fbe18bc4b..5d957ce26b 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -424,7 +424,9 @@ class TokenController(Application): token_id=token) user_ref = old_token_ref['user'] - assert tenant_id in user_ref['tenants'] + tenants = self.identity_api.get_tenants_for_user(context, + user_ref['id']) + assert tenant_id in tenants tenant_ref = self.identity_api.get_tenant(context=context, tenant_id=tenant_id) From 2d6b348e2738ae97e858d236847436ccaba942f0 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 14:56:19 -0800 Subject: [PATCH 170/334] add role refs to validate token --- keystone/service.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/keystone/service.py b/keystone/service.py index 5d957ce26b..297b14a957 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -465,22 +465,21 @@ class TokenController(Application): """ # TODO(termie): this stuff should probably be moved to middleware - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['metadata'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) + self.assert_admin(context) token_ref = self.token_api.get_token(context=context, token_id=token_id) if belongs_to: assert token_ref['tenant']['id'] == belongs_to - return self._format_token(token_ref) + + # TODO(termie): optimize this call at some point and put it into the + # the return for metadata + # fill out the roles in the metadata + metadata_ref = token_ref['metadata'] + roles_ref = [] + for role_id in metadata_ref.get('roles', []): + roles_ref.append(self.identity_api.get_role(context, role_id)) + return self._format_token(token_ref, roles_ref) def endpoints(self, context, token_id): """Return service catalog endpoints.""" From 2a31259fa0afa504a523fc40f56027f22e540533 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 14:58:29 -0800 Subject: [PATCH 171/334] TODO --- TODO | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000000..2e781a996a --- /dev/null +++ b/TODO @@ -0,0 +1,2 @@ +- test authenticate with token only +- test validate token From be52a5efef83babe6ed841f3eba052fa3b4e0945 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 16:09:44 -0800 Subject: [PATCH 172/334] strip newlines --- keystone/backends/templated.py | 2 +- keystone/wsgi.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/keystone/backends/templated.py b/keystone/backends/templated.py index f480459e99..989ee0f679 100644 --- a/keystone/backends/templated.py +++ b/keystone/backends/templated.py @@ -50,7 +50,7 @@ class TemplatedCatalog(kvs.KvsCatalog): if ' = ' not in line: continue - k, v = line.split(' = ') + k, v = line.strip().split(' = ') if not k.startswith('catalog.'): continue diff --git a/keystone/wsgi.py b/keystone/wsgi.py index 401b0f0136..ee14970cb2 100644 --- a/keystone/wsgi.py +++ b/keystone/wsgi.py @@ -246,7 +246,8 @@ class Debug(Middleware): """Iterator that prints the contents of a wrapper string.""" logging.debug('%s %s %s', ('*' * 20), 'RESPONSE BODY', ('*' * 20)) for part in app_iter: - sys.stdout.write(part) + #sys.stdout.write(part) + logging.debug(part) #sys.stdout.flush() yield part print From 8ea6e8f497377cc06a334b9e7fa82a5b87b92f47 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 11 Jan 2012 16:45:08 -0800 Subject: [PATCH 173/334] add some more todos --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 2e781a996a..db7f242d70 100644 --- a/TODO +++ b/TODO @@ -1,2 +1,5 @@ - test authenticate with token only - test validate token +- policy tests +- ec2 support + From 1bd1349482548909416bead8b4dffe94af2bac81 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 12:45:08 -0800 Subject: [PATCH 174/334] add a couple more tests --- TODO | 1 - keystone/service.py | 26 ++++++++++++++++---------- tests/test_keystoneclient.py | 24 ++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/TODO b/TODO index db7f242d70..6eb577b119 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,3 @@ -- test authenticate with token only - test validate token - policy tests - ec2 support diff --git a/keystone/service.py b/keystone/service.py index 297b14a957..b194e267e1 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -426,24 +426,30 @@ class TokenController(Application): tenants = self.identity_api.get_tenants_for_user(context, user_ref['id']) - assert tenant_id in tenants + if tenant_id: + assert tenant_id in tenants tenant_ref = self.identity_api.get_tenant(context=context, tenant_id=tenant_id) - metadata_ref = self.identity_api.get_metadata( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id']) + if tenant_ref: + metadata_ref = self.identity_api.get_metadata( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id']) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + metadata=metadata_ref) + else: + metadata_ref = {} + catalog_ref = {} + token_ref = self.token_api.create_token( context, dict(expires='', user=user_ref, tenant=tenant_ref, metadata=metadata_ref)) - catalog_ref = self.catalog_api.get_catalog( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id'], - metadata=metadata_ref) # TODO(termie): optimize this call at some point and put it into the # the return for metadata diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 8073a00619..bb568b0963 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -76,7 +76,31 @@ class KcMasterTestCase(CompatTestCase): self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_tenant_id_and_tenants(self): + client = self._client(username='FOO', + password='foo2', + tenant_id='bar') + + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + def test_authenticate_token_no_tenant(self): client = self.foo_client() + token = client.auth_token + token_client = self._client(token=token) + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + def test_authenticate_token_tenant_id(self): + client = self.foo_client() + token = client.auth_token + token_client = self._client(token=token, tenant_id='bar') + tenants = client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + def test_authenticate_token_tenant_name(self): + client = self.foo_client() + token = client.auth_token + token_client = self._client(token=token, tenant_name='BAR') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) From deab5c450692fd2f1ec97ef61bf9143b35007fe4 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 13:23:57 -0800 Subject: [PATCH 175/334] dynamic manager classes for now --- keystone/backends/kvs.py | 80 +++++++++++++------------- keystone/backends/sql/core.py | 30 +++++----- keystone/bufferedhttp.py | 2 + keystone/catalog.py | 24 +------- keystone/config.py | 1 + keystone/identity.py | 103 ++-------------------------------- keystone/manager.py | 23 ++++++++ keystone/policy.py | 12 +--- keystone/service.py | 25 +++++---- keystone/token.py | 25 +-------- 10 files changed, 109 insertions(+), 216 deletions(-) create mode 100644 keystone/manager.py diff --git a/keystone/backends/kvs.py b/keystone/backends/kvs.py index 5e3d00fc21..c67eac05ea 100644 --- a/keystone/backends/kvs.py +++ b/keystone/backends/kvs.py @@ -115,48 +115,48 @@ class KvsIdentity(object): self.update_metadata(user_id, tenant_id, metadata_ref) # CRUD - def create_user(self, id, user): - self.db.set('user-%s' % id, user) + def create_user(self, user_id, user): + self.db.set('user-%s' % user_id, user) self.db.set('user_name-%s' % user['name'], user) user_list = set(self.db.get('user_list', [])) - user_list.add(id) + user_list.add(user_id) self.db.set('user_list', list(user_list)) return user - def update_user(self, id, user): + def update_user(self, user_id, user): # get the old name and delete it too - old_user = self.db.get('user-%s' % id) + old_user = self.db.get('user-%s' % user_id) self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % id, user) + self.db.set('user-%s' % user_id, user) self.db.set('user_name-%s' % user['name'], user) return user - def delete_user(self, id): - old_user = self.db.get('user-%s' % id) + def delete_user(self, user_id): + old_user = self.db.get('user-%s' % user_id) self.db.delete('user_name-%s' % old_user['name']) - self.db.delete('user-%s' % id) + self.db.delete('user-%s' % user_id) user_list = set(self.db.get('user_list', [])) - user_list.remove(id) + user_list.remove(user_id) self.db.set('user_list', list(user_list)) return None - def create_tenant(self, id, tenant): - self.db.set('tenant-%s' % id, tenant) + def create_tenant(self, tenant_id, tenant): + self.db.set('tenant-%s' % tenant_id, tenant) self.db.set('tenant_name-%s' % tenant['name'], tenant) return tenant - def update_tenant(self, id, tenant): + def update_tenant(self, tenant_id, tenant): # get the old name and delete it too - old_tenant = self.db.get('tenant-%s' % id) + old_tenant = self.db.get('tenant-%s' % tenant_id) self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant-%s' % tenant_id, tenant) self.db.set('tenant_name-%s' % tenant['name'], tenant) return tenant - def delete_tenant(self, id): - old_tenant = self.db.get('tenant-%s' % id) + def delete_tenant(self, tenant_id): + old_tenant = self.db.get('tenant-%s' % tenant_id) self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.delete('tenant-%s' % id) + self.db.delete('tenant-%s' % tenant_id) return None def create_metadata(self, user_id, tenant_id, metadata): @@ -171,21 +171,21 @@ class KvsIdentity(object): self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) return None - def create_role(self, id, role): - self.db.set('role-%s' % id, role) + def create_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) role_list = set(self.db.get('role_list', [])) - role_list.add(id) + role_list.add(role_id) self.db.set('role_list', list(role_list)) return role - def update_role(self, id, role): - self.db.set('role-%s' % id, role) + def update_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) return role - def delete_role(self, id): - self.db.delete('role-%s' % id) + def delete_role(self, role_id): + self.db.delete('role-%s' % role_id) role_list = set(self.db.get('role_list', [])) - role_list.remove(id) + role_list.remove(role_id) self.db.set('role_list', list(role_list)) return None @@ -199,15 +199,15 @@ class KvsToken(object): self.db = db # Public interface - def get_token(self, id): - return self.db.get('token-%s' % id) + def get_token(self, token_id): + return self.db.get('token-%s' % token_id) - def create_token(self, id, data): - self.db.set('token-%s' % id, data) + def create_token(self, token_id, data): + self.db.set('token-%s' % token_id, data) return data - def delete_token(self, id): - return self.db.delete('token-%s' % id) + def delete_token(self, token_id): + return self.db.delete('token-%s' % token_id) class KvsCatalog(object): @@ -228,21 +228,21 @@ class KvsCatalog(object): def list_services(self): return self.db.get('service_list', []) - def create_service(self, id, service): - self.db.set('service-%s' % id, service) + def create_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) service_list = set(self.db.get('service_list', [])) - service_list.add(id) + service_list.add(service_id) self.db.set('service_list', list(service_list)) return service - def update_service(self, id, service): - self.db.set('service-%s' % id, service) + def update_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) return service - def delete_service(self, id): - self.db.delete('service-%s' % id) + def delete_service(self, service_id): + self.db.delete('service-%s' % service_id) service_list = set(self.db.get('service_list', [])) - service_list.remove(id) + service_list.remove(service_id) self.db.set('service_list', list(service_list)) return None diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index e6862c1f03..c17f991284 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -347,7 +347,7 @@ class SqlIdentity(SqlBase): self.create_metadata(user_id, tenant_id, metadata_ref) # CRUD - def create_user(self, id, user): + def create_user(self, user_id, user): session = self.get_session() with session.begin(): user_ref = User.from_dict(user) @@ -355,10 +355,10 @@ class SqlIdentity(SqlBase): session.flush() return user_ref.to_dict() - def update_user(self, id, user): + def update_user(self, user_id, user): session = self.get_session() with session.begin(): - user_ref = session.query(User).filter_by(id=id).first() + user_ref = session.query(User).filter_by(id=user_id).first() old_user_dict = user_ref.to_dict() for k in user: old_user_dict[k] = user[k] @@ -369,14 +369,14 @@ class SqlIdentity(SqlBase): session.flush() return user_ref - def delete_user(self, id): + def delete_user(self, user_id): session = self.get_session() - user_ref = session.query(User).filter_by(id=id).first() + user_ref = session.query(User).filter_by(id=user_id).first() with session.begin(): session.delete(user_ref) session.flush() - def create_tenant(self, id, tenant): + def create_tenant(self, tenant_id, tenant): session = self.get_session() with session.begin(): tenant_ref = Tenant.from_dict(tenant) @@ -384,10 +384,10 @@ class SqlIdentity(SqlBase): session.flush() return tenant_ref.to_dict() - def update_tenant(self, id, tenant): + def update_tenant(self, tenant_id, tenant): session = self.get_session() with session.begin(): - tenant_ref = session.query(Tenant).filter_by(id=id).first() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() old_tenant_dict = tenant_ref.to_dict() for k in tenant: old_tenant_dict[k] = tenant[k] @@ -398,9 +398,9 @@ class SqlIdentity(SqlBase): session.flush() return tenant_ref - def delete_tenant(self, id): + def delete_tenant(self, tenant_id): session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=id).first() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() with session.begin(): session.delete(tenant_ref) session.flush() @@ -432,25 +432,25 @@ class SqlIdentity(SqlBase): self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) return None - def create_role(self, id, role): + def create_role(self, role_id, role): session = self.get_session() with session.begin(): session.add(Role(**role)) session.flush() return role - def update_role(self, id, role): + def update_role(self, role_id, role): session = self.get_session() with session.begin(): - role_ref = session.query(Role).filter_by(id=id).first() + role_ref = session.query(Role).filter_by(id=role_id).first() for k in role: role_ref[k] = role[k] session.flush() return role_ref - def delete_role(self, id): + def delete_role(self, role_id): session = self.get_session() - role_ref = session.query(Role).filter_by(id=id).first() + role_ref = session.query(Role).filter_by(id=role_id).first() with session.begin(): session.delete(role_ref) diff --git a/keystone/bufferedhttp.py b/keystone/bufferedhttp.py index fdb35ee657..769a9b8bf3 100644 --- a/keystone/bufferedhttp.py +++ b/keystone/bufferedhttp.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/keystone/catalog.py b/keystone/catalog.py index 6ad348e5b9..8382108d1f 100644 --- a/keystone/catalog.py +++ b/keystone/catalog.py @@ -1,30 +1,12 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# the catalog interfaces - from keystone import config -from keystone import utils +from keystone import manager CONF = config.CONF -class Manager(object): +class Manager(manager.Manager): def __init__(self): - self.driver = utils.import_object(CONF.catalog.driver) - - def get_catalog(self, context, user_id, tenant_id, metadata=None): - """Return info for a catalog if it is valid.""" - return self.driver.get_catalog(user_id, tenant_id, metadata=metadata) - - def get_service(self, context, service_id): - return self.driver.get_service(service_id) - - def list_services(self, context): - return self.driver.list_services() - - def create_service(self, context, service_id, data): - return self.driver.create_service(service_id, data) - - def delete_service(self, context, service_id): - return self.driver.delete_service(service_id) + super(Manager, self).__init__(CONF.catalog.driver) diff --git a/keystone/config.py b/keystone/config.py index c99a6aff19..6cbc1571b8 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -1,4 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 + import gettext import logging import sys diff --git a/keystone/identity.py b/keystone/identity.py index 60bada9f3f..6ef8298a6a 100644 --- a/keystone/identity.py +++ b/keystone/identity.py @@ -1,105 +1,12 @@ -# these will be the basic data types for tenants and users -# backends will make use of them to return something that conforms to their -# apis +# vim: tabstop=4 shiftwidth=4 softtabstop=4 from keystone import config -from keystone import utils +from keystone import manager CONF = config.CONF -class Manager(object): - def __init__(self): - self.driver = utils.import_object(CONF.identity.driver) - - def authenticate(self, context, **kwargs): - """Passthru authentication to the identity driver. - - This call will basically just result in getting a token. - """ - return self.driver.authenticate(**kwargs) - - def get_user(self, context, user_id): - return self.driver.get_user(user_id) - - def get_user_by_name(self, context, user_name): - return self.driver.get_user_by_name(user_name) - - def get_tenant(self, context, tenant_id): - return self.driver.get_tenant(tenant_id) - - def get_tenant_by_name(self, context, tenant_name): - return self.driver.get_tenant_by_name(tenant_name) - - def get_metadata(self, context, user_id, tenant_id): - return self.driver.get_metadata(user_id, tenant_id) - - def get_role(self, context, role_id): - return self.driver.get_role(role_id) - - # NOTE(termie): i think it will probably be a bad move in the end to try to - # list all users - def list_users(self, context): - return self.driver.list_users() - - def list_roles(self, context): - return self.driver.list_roles() - - # These should probably be the high-level API calls - def add_user_to_tenant(self, context, user_id, tenant_id): - self.driver.add_user_to_tenant(user_id, tenant_id) - - def remove_user_from_tenant(self, context, user_id, tenant_id): - self.driver.remove_user_from_tenant(user_id, tenant_id) - - def get_tenants_for_user(self, context, user_id): - return self.driver.get_tenants_for_user(user_id) - - def get_roles_for_user_and_tenant(self, context, user_id, tenant_id): - return self.driver.get_roles_for_user_and_tenant(user_id, tenant_id) - - def add_role_to_user_and_tenant(self, context, user_id, tenant_id, role_id): - return self.driver.add_role_to_user_and_tenant(user_id, tenant_id, role_id) - - def remove_role_from_user_and_tenant(self, context, user_id, - tenant_id, role_id): - return self.driver.remove_role_from_user_and_tenant( - user_id, tenant_id, role_id) - - # CRUD operations - def create_user(self, context, user_id, data): - return self.driver.create_user(user_id, data) - - def update_user(self, context, user_id, data): - return self.driver.update_user(user_id, data) - - def delete_user(self, context, user_id): - return self.driver.delete_user(user_id) - - def create_tenant(self, context, tenant_id, data): - return self.driver.create_tenant(tenant_id, data) - - def update_tenant(self, context, tenant_id, data): - return self.driver.update_tenant(tenant_id, data) - - def delete_tenant(self, context, tenant_id): - return self.driver.delete_tenant(tenant_id) - - def create_metadata(self, context, user_id, tenant_id, data): - return self.driver.create_metadata(user_id, tenant_id, data) - - def update_metadata(self, context, user_id, tenant_id, data): - return self.driver.update_metadata(user_id, tenant_id, data) - - def delete_metadata(self, context, user_id, tenant_id): - return self.driver.delete_metadata(user_id, tenant_id) - - def create_role(self, context, role_id, data): - return self.driver.create_role(role_id, data) - - def update_role(self, context, role_id, data): - return self.driver.update_role(role_id, data) - - def delete_role(self, context, role_id): - return self.driver.delete_role(role_id) +class Manager(manager.Manager): + def __init__(self): + super(Manager, self).__init__(CONF.identity.driver) diff --git a/keystone/manager.py b/keystone/manager.py new file mode 100644 index 0000000000..566d2a23ba --- /dev/null +++ b/keystone/manager.py @@ -0,0 +1,23 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import functools + +from keystone import config +from keystone import utils + + +class Manager(object): + def __init__(self, driver_name): + self.driver = utils.import_object(driver_name) + + def __getattr__(self, name): + # NOTE(termie): context is the first argument, we're going to strip + # that for now, in the future we'll probably do some + # logging and whatnot in this class + f = getattr(self.driver, name) + + @functools.wraps(f) + def _wrapper(context, *args, **kw): + return f(*args, **kw) + setattr(self, name, _wrapper) + return _wrapper diff --git a/keystone/policy.py b/keystone/policy.py index f5c3c6d4c5..f41f54ad78 100644 --- a/keystone/policy.py +++ b/keystone/policy.py @@ -1,18 +1,12 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# the catalog interfaces - from keystone import config -from keystone import utils +from keystone import manager CONF = config.CONF -class Manager(object): +class Manager(manager.Manager): def __init__(self): - self.driver = utils.import_object(CONF.policy.driver) - - def can_haz(self, context, target, credentials): - """Check whether the given creds can perform action on target.""" - return self.driver.can_haz(target, credentials) + super(Manager, self).__init__(CONF.policy.driver) diff --git a/keystone/service.py b/keystone/service.py index b194e267e1..dc666dc6c3 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -368,6 +368,7 @@ class TokenController(Application): that will return a token that is scoped to that tenant. """ + token_id = uuid.uuid4().hex if 'passwordCredentials' in auth: username = auth['passwordCredentials'].get('username', '') password = auth['passwordCredentials'].get('password', '') @@ -394,10 +395,11 @@ class TokenController(Application): password=password, tenant_id=tenant_id) token_ref = self.token_api.create_token( - context, dict(expires='', - user=user_ref, - tenant=tenant_ref, - metadata=metadata_ref)) + context, token_id, dict(expires='', + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) if tenant_ref: catalog_ref = self.catalog_api.get_catalog( context=context, @@ -446,10 +448,11 @@ class TokenController(Application): catalog_ref = {} token_ref = self.token_api.create_token( - context, dict(expires='', - user=user_ref, - tenant=tenant_ref, - metadata=metadata_ref)) + context, token_id, dict(expires='', + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) # TODO(termie): optimize this call at some point and put it into the # the return for metadata @@ -625,18 +628,18 @@ class TenantController(Application): tenant_ref['id'] = tenant_id tenant = self.identity_api.create_tenant( - context, tenant_id=tenant_id, data=tenant_ref) + context, tenant_id, tenant_ref) return {'tenant': tenant} def update_tenant(self, context, tenant_id, tenant): self.assert_admin(context) tenant_ref = self.identity_api.update_tenant( - context, tenant_id=tenant_id, data=tenant) + context, tenant_id, tenant) return {'tenant': tenant_ref} def delete_tenant(self, context, tenant_id, **kw): self.assert_admin(context) - self.identity_api.delete_tenant(context, tenant_id=tenant_id) + self.identity_api.delete_tenant(context, tenant_id) def get_tenant_users(self, context, **kw): self.assert_admin(context) diff --git a/keystone/token.py b/keystone/token.py index 194767bc8a..0657f8fac9 100644 --- a/keystone/token.py +++ b/keystone/token.py @@ -1,31 +1,12 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# the token interfaces - -import uuid - from keystone import config -from keystone import logging -from keystone import utils +from keystone import manager CONF = config.CONF -class Manager(object): +class Manager(manager.Manager): def __init__(self): - self.driver = utils.import_object(CONF.token.driver) - - def create_token(self, context, data): - token = uuid.uuid4().hex - data['id'] = token - token_ref = self.driver.create_token(token, data) - return token_ref - - @logging.log_debug - def get_token(self, context, token_id): - """Return info for a token if it is valid.""" - return self.driver.get_token(token_id) - - def delete_token(self, context, token_id): - self.driver.delete_token(token_id) + super(Manager, self).__init__(CONF.token.driver) From 7b0f71bce4645447f6aa342cb81d554d6b19d3de Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 14:28:22 -0800 Subject: [PATCH 176/334] add some docs to manager --- keystone/manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/keystone/manager.py b/keystone/manager.py index 566d2a23ba..d6d1f602d6 100644 --- a/keystone/manager.py +++ b/keystone/manager.py @@ -7,10 +7,23 @@ from keystone import utils class Manager(object): + """Base class for intermediary request layer. + + The Manager layer exists to support additional logic that applies to all + or some of the methods exposed by a service that are not specific to the + HTTP interface. + + It also provides a stable entry point to dynamic backends. + + An example of a probable use case is logging all the calls. + + """ + def __init__(self, driver_name): self.driver = utils.import_object(driver_name) def __getattr__(self, name): + """Forward calls to the underlying driver.""" # NOTE(termie): context is the first argument, we're going to strip # that for now, in the future we'll probably do some # logging and whatnot in this class From 6a48676db101b18020d90fe4e1ab6d0f016a3dd4 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 14:35:17 -0800 Subject: [PATCH 177/334] remove models.py --- keystone/backends/sql/core.py | 1 - keystone/logging.py | 4 +++ keystone/models.py | 28 ------------------- keystone/service.py | 1 - keystone/test.py | 1 - tests/test_backend_kvs.py | 1 - tests/test_backend_sql.py | 1 - tests/test_legacy_compat.py | 49 ++++++++++++++++----------------- tests/test_novaclient_compat.py | 1 - 9 files changed, 28 insertions(+), 59 deletions(-) delete mode 100644 keystone/models.py diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index c17f991284..b75025313d 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -11,7 +11,6 @@ import sqlalchemy.pool import sqlalchemy.engine.url from keystone import config -from keystone import models from keystone.backends.sql import migration diff --git a/keystone/logging.py b/keystone/logging.py index 84e3e66d30..7ef073763b 100644 --- a/keystone/logging.py +++ b/keystone/logging.py @@ -1,3 +1,7 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +"""Wrapper for built-in logging module.""" + from __future__ import absolute_import import functools diff --git a/keystone/models.py b/keystone/models.py deleted file mode 100644 index 18402bbd3e..0000000000 --- a/keystone/models.py +++ /dev/null @@ -1,28 +0,0 @@ - - -class Token(dict): - def __init__(self, id=None, user=None, tenant=None, *args, **kw): - super(Token, self).__init__(id=id, user=user, tenant=tenant, *args, **kw) - - -class User(dict): - def __init__(self, id=None, *args, **kw): - super(User, self).__init__(id=id, *args, **kw) - - -class Tenant(dict): - def __init__(self, id=None, *args, **kw): - super(Tenant, self).__init__(id=id, *args, **kw) - - -class Role(dict): - def __init__(self, id=None, *args, **kw): - super(Role, self).__init__(id=id, *args, **kw) - - -class Metadata(dict): - def __init__(self, user_id=None, tenant_id=None, *args, **kw): - super(Metadata, self).__init__(user_id=user_id, - tenant_id=tenant_id, - *args, - **kw) diff --git a/keystone/service.py b/keystone/service.py index dc666dc6c3..b66fafb9e8 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -1,6 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# this is the web public frontend that emulates keystone import json import urllib import urlparse diff --git a/keystone/test.py b/keystone/test.py index 6b100d77f3..2288e5fa4e 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -10,7 +10,6 @@ from keystone import catalog from keystone import config from keystone import identity from keystone import logging -from keystone import models from keystone import token from keystone import utils from keystone import wsgi diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index a5a67ff2d3..85b9fef6c7 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -1,6 +1,5 @@ import uuid -from keystone import models from keystone import test from keystone.backends import kvs diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index 24cb4a81b2..faaa3b21df 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -2,7 +2,6 @@ import os import uuid from keystone import config -from keystone import models from keystone import test from keystone.backends import sql from keystone.backends.sql import util as sql_util diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index fef2b8a46f..fca00b6848 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -7,7 +7,6 @@ from nose import exc from keystone import config from keystone import logging -from keystone import models from keystone import test from keystone import utils @@ -52,13 +51,13 @@ class CompatTestCase(test.TestCase): # validate_token call self.tenant_345 = self.identity_api.create_tenant( '345', - models.Tenant(id='345', name='My Project')) + dict(id='345', name='My Project')) self.user_123 = self.identity_api.create_user( '123', - models.User(id='123', - name='jqsmith', - tenants=[self.tenant_345['id']], - password='password')) + dict(id='123', + name='jqsmith', + tenants=[self.tenant_345['id']], + password='password')) self.metadata_123 = self.identity_api.create_metadata( self.user_123['id'], self.tenant_345['id'], dict(roles=[{'id': '234', @@ -69,11 +68,11 @@ class CompatTestCase(test.TestCase): roles_links=[])) self.token_123 = self.token_api.create_token( 'ab48a9efdfedb23ty3494', - models.Token(id='ab48a9efdfedb23ty3494', - expires='2010-11-01T03:32:15-05:00', - user=self.user_123, - tenant=self.tenant_345, - metadata=self.metadata_123)) + dict(id='ab48a9efdfedb23ty3494', + expires='2010-11-01T03:32:15-05:00', + user=self.user_123, + tenant=self.tenant_345, + metadata=self.metadata_123)) # auth call # NOTE(termie): the service catalog in the sample doesn't really have @@ -90,29 +89,29 @@ class CompatTestCase(test.TestCase): # tenants_for_token call self.user_foo = self.identity_api.create_user( 'foo', - models.User(id='foo', name='FOO', tenants=['1234', '3456'])) + dict(id='foo', name='FOO', tenants=['1234', '3456'])) self.tenant_1234 = self.identity_api.create_tenant( '1234', - models.Tenant(id='1234', - name='ACME Corp', - description='A description ...', - enabled=True)) + dict(id='1234', + name='ACME Corp', + description='A description ...', + enabled=True)) self.tenant_3456 = self.identity_api.create_tenant( '3456', - models.Tenant(id='3456', - name='Iron Works', - description='A description ...', - enabled=True)) + dict(id='3456', + name='Iron Works', + description='A description ...', + enabled=True)) self.token_foo_unscoped = self.token_api.create_token( 'foo_unscoped', - models.Token(id='foo_unscoped', - user=self.user_foo)) + dict(id='foo_unscoped', + user=self.user_foo)) self.token_foo_scoped = self.token_api.create_token( 'foo_scoped', - models.Token(id='foo_scoped', - user=self.user_foo, - tenant=self.tenant_1234)) + dict(id='foo_scoped', + user=self.user_foo, + tenant=self.tenant_1234)) class DiabloCompatTestCase(CompatTestCase): diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 8aa6e344af..203fa9399e 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -5,7 +5,6 @@ import sys from keystone import config from keystone import logging -from keystone import models from keystone import test from keystone import utils From 9d7c5c05d2b9cbac96f8afdf014f2dc9158c48b7 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 15:23:39 -0800 Subject: [PATCH 178/334] re-indent test.py --- keystone/test.py | 323 ++++++++++++++++++++++++----------------------- 1 file changed, 163 insertions(+), 160 deletions(-) diff --git a/keystone/test.py b/keystone/test.py index 2288e5fa4e..7ef1b6ede0 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + import os import unittest import subprocess @@ -26,207 +28,208 @@ cd = os.chdir def rootdir(*p): - return os.path.join(ROOTDIR, *p) + return os.path.join(ROOTDIR, *p) def etcdir(*p): - return os.path.join(ETCDIR, *p) + return os.path.join(ETCDIR, *p) def testsdir(*p): - return os.path.join(TESTSDIR, *p) + return os.path.join(TESTSDIR, *p) def checkout_vendor(repo, rev): - name = repo.split('/')[-1] - if name.endswith('.git'): - name = name[:-4] + name = repo.split('/')[-1] + if name.endswith('.git'): + name = name[:-4] - revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) - modcheck = os.path.join(VENDOR, '.%s-%s' % (name, rev.replace('/', '_'))) - try: - if os.path.exists(modcheck): - mtime = os.stat(modcheck).st_mtime - if int(time.time()) - mtime < 10000: - return revdir + revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) + modcheck = os.path.join(VENDOR, '.%s-%s' % (name, rev.replace('/', '_'))) + try: + if os.path.exists(modcheck): + mtime = os.stat(modcheck).st_mtime + if int(time.time()) - mtime < 10000: + return revdir - if not os.path.exists(revdir): - utils.git('clone', repo, revdir) + if not os.path.exists(revdir): + utils.git('clone', repo, revdir) - cd(revdir) - utils.git('pull') - utils.git('checkout', '-q', rev) + cd(revdir) + utils.git('pull') + utils.git('checkout', '-q', rev) - # write out a modified time - with open(modcheck, 'w') as fd: - fd.write('1') - except subprocess.CalledProcessError as e: - logging.warning('Failed to checkout %s', repo) - pass - return revdir + # write out a modified time + with open(modcheck, 'w') as fd: + fd.write('1') + except subprocess.CalledProcessError as e: + logging.warning('Failed to checkout %s', repo) + return revdir class TestClient(object): - def __init__(self, app=None, token=None): - self.app = app - self.token = token + def __init__(self, app=None, token=None): + self.app = app + self.token = token - def request(self, method, path, headers=None, body=None): - if headers is None: - headers = {} + def request(self, method, path, headers=None, body=None): + if headers is None: + headers = {} - if self.token: - headers.setdefault('X-Auth-Token', self.token) + if self.token: + headers.setdefault('X-Auth-Token', self.token) - req = wsgi.Request.blank(path) - req.method = method - for k, v in headers.iteritems(): - req.headers[k] = v - if body: - req.body = body - return req.get_response(self.app) + req = wsgi.Request.blank(path) + req.method = method + for k, v in headers.iteritems(): + req.headers[k] = v + if body: + req.body = body + return req.get_response(self.app) - def get(self, path, headers=None): - return self.request('GET', path=path, headers=headers) + def get(self, path, headers=None): + return self.request('GET', path=path, headers=headers) - def post(self, path, headers=None, body=None): - return self.request('POST', path=path, headers=headers, body=body) + def post(self, path, headers=None, body=None): + return self.request('POST', path=path, headers=headers, body=body) - def put(self, path, headers=None, body=None): - return self.request('PUT', path=path, headers=headers, body=body) + def put(self, path, headers=None, body=None): + return self.request('PUT', path=path, headers=headers, body=body) class TestCase(unittest.TestCase): - def __init__(self, *args, **kw): - super(TestCase, self).__init__(*args, **kw) - self._paths = [] - self._memo = {} + def __init__(self, *args, **kw): + super(TestCase, self).__init__(*args, **kw) + self._paths = [] + self._memo = {} - def setUp(self): - super(TestCase, self).setUp() + def setUp(self): + super(TestCase, self).setUp() - def tearDown(self): - for path in self._paths: - if path in sys.path: - sys.path.remove(path) - CONF.reset() - super(TestCase, self).tearDown() + def tearDown(self): + for path in self._paths: + if path in sys.path: + sys.path.remove(path) + CONF.reset() + super(TestCase, self).tearDown() - def load_backends(self): - """Hacky shortcut to load the backends for data manipulation.""" - self.identity_api = utils.import_object(CONF.identity.driver) - self.token_api = utils.import_object(CONF.token.driver) - self.catalog_api = utils.import_object(CONF.catalog.driver) + def load_backends(self): + """Hacky shortcut to load the backends for data manipulation.""" + self.identity_api = utils.import_object(CONF.identity.driver) + self.token_api = utils.import_object(CONF.token.driver) + self.catalog_api = utils.import_object(CONF.catalog.driver) - def load_fixtures(self, fixtures): - """Hacky basic and naive fixture loading based on a python module. + def load_fixtures(self, fixtures): + """Hacky basic and naive fixture loading based on a python module. - Expects that the various APIs into the various services are already - defined on `self`. + Expects that the various APIs into the various services are already + defined on `self`. - """ - # TODO(termie): doing something from json, probably based on Django's - # loaddata will be much preferred. - for tenant in fixtures.TENANTS: - rv = self.identity_api.create_tenant(tenant['id'], tenant) - setattr(self, 'tenant_%s' % tenant['id'], rv) + """ + # TODO(termie): doing something from json, probably based on Django's + # loaddata will be much preferred. + for tenant in fixtures.TENANTS: + rv = self.identity_api.create_tenant(tenant['id'], tenant) + setattr(self, 'tenant_%s' % tenant['id'], rv) - for user in fixtures.USERS: - user_copy = user.copy() - tenants = user_copy.pop('tenants') - rv = self.identity_api.create_user(user['id'], user_copy) - for tenant_id in tenants: - self.identity_api.add_user_to_tenant(tenant_id, user['id']) - setattr(self, 'user_%s' % user['id'], rv) + for user in fixtures.USERS: + user_copy = user.copy() + tenants = user_copy.pop('tenants') + rv = self.identity_api.create_user(user['id'], user_copy) + for tenant_id in tenants: + self.identity_api.add_user_to_tenant(tenant_id, user['id']) + setattr(self, 'user_%s' % user['id'], rv) - for role in fixtures.ROLES: - rv = self.identity_api.create_role(role['id'], role) - setattr(self, 'role_%s' % role['id'], rv) + for role in fixtures.ROLES: + rv = self.identity_api.create_role(role['id'], role) + setattr(self, 'role_%s' % role['id'], rv) - for metadata in fixtures.METADATA: - metadata_ref = metadata.copy() - # TODO(termie): these will probably end up in the model anyway, so this - # may be futile - del metadata_ref['user_id'] - del metadata_ref['tenant_id'] - rv = self.identity_api.create_metadata( - metadata['user_id'], metadata['tenant_id'], metadata_ref) - setattr(self, - 'metadata_%s%s' % (metadata['user_id'], - metadata['tenant_id']), rv) + for metadata in fixtures.METADATA: + metadata_ref = metadata.copy() + # TODO(termie): these will probably end up in the model anyway, + # so this may be futile + del metadata_ref['user_id'] + del metadata_ref['tenant_id'] + rv = self.identity_api.create_metadata(metadata['user_id'], + metadata['tenant_id'], + metadata_ref) + setattr(self, + 'metadata_%s%s' % (metadata['user_id'], + metadata['tenant_id']), rv) - def _paste_config(self, config): - if not config.startswith('config:'): - test_path = os.path.join(TESTSDIR, config) - etc_path = os.path.join(ROOTDIR, 'etc', config) - for path in [test_path, etc_path]: - if os.path.exists('%s.conf' % path): - return 'config:%s.conf' % path - return config + def _paste_config(self, config): + if not config.startswith('config:'): + test_path = os.path.join(TESTSDIR, config) + etc_path = os.path.join(ROOTDIR, 'etc', config) + for path in [test_path, etc_path]: + if os.path.exists('%s.conf' % path): + return 'config:%s.conf' % path + return config - def loadapp(self, config, name='main'): - return deploy.loadapp(self._paste_config(config), name=name) + def loadapp(self, config, name='main'): + return deploy.loadapp(self._paste_config(config), name=name) - def appconfig(self, config): - return deploy.appconfig(self._paste_config(config)) + def appconfig(self, config): + return deploy.appconfig(self._paste_config(config)) - def serveapp(self, config, name=None): - app = self.loadapp(config, name=name) - server = wsgi.Server(app, 0) - server.start(key='socket') + def serveapp(self, config, name=None): + app = self.loadapp(config, name=name) + server = wsgi.Server(app, 0) + server.start(key='socket') - # Service catalog tests need to know the port we ran on. - port = server.socket_info['socket'][1] - CONF.public_port = port - CONF.admin_port = port - return server + # Service catalog tests need to know the port we ran on. + port = server.socket_info['socket'][1] + CONF.public_port = port + CONF.admin_port = port + return server - def client(self, app, *args, **kw): - return TestClient(app, *args, **kw) + def client(self, app, *args, **kw): + return TestClient(app, *args, **kw) - def add_path(self, path): - sys.path.insert(0, path) - self._paths.append(path) + def add_path(self, path): + sys.path.insert(0, path) + self._paths.append(path) - def assertListEquals(self, expected, actual): - copy = expected[:] - #print expected, actual - self.assertEquals(len(expected), len(actual)) - while copy: - item = copy.pop() - matched = False - for x in actual: - #print 'COMPARE', item, x, + def assertListEquals(self, expected, actual): + copy = expected[:] + #print expected, actual + self.assertEquals(len(expected), len(actual)) + while copy: + item = copy.pop() + matched = False + for x in actual: + #print 'COMPARE', item, x, + try: + self.assertDeepEquals(item, x) + matched = True + #print 'MATCHED' + break + except AssertionError as e: + #print e + pass + if not matched: + raise AssertionError('Expected: %s\n Got: %s' % (expected, + actual)) + + def assertDictEquals(self, expected, actual): + for k in expected: + self.assertTrue(k in actual, + "Expected key %s not in %s." % (k, actual)) + self.assertDeepEquals(expected[k], actual[k]) + + for k in actual: + self.assertTrue(k in expected, + "Unexpected key %s in %s." % (k, actual)) + + def assertDeepEquals(self, expected, actual): try: - self.assertDeepEquals(item, x) - matched = True - #print 'MATCHED' - break + if type(expected) is type([]) or type(expected) is type(tuple()): + # assert items equal, ignore order + self.assertListEquals(expected, actual) + elif type(expected) is type({}): + self.assertDictEquals(expected, actual) + else: + self.assertEquals(expected, actual) except AssertionError as e: - #print e - pass - if not matched: - raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) - - def assertDictEquals(self, expected, actual): - for k in expected: - self.assertTrue(k in actual, - "Expected key %s not in %s." % (k, actual)) - self.assertDeepEquals(expected[k], actual[k]) - - for k in actual: - self.assertTrue(k in expected, - "Unexpected key %s in %s." % (k, actual)) - - def assertDeepEquals(self, expected, actual): - try: - if type(expected) is type([]) or type(expected) is type(tuple()): - # assert items equal, ignore order - self.assertListEquals(expected, actual) - elif type(expected) is type({}): - self.assertDictEquals(expected, actual) - else: - self.assertEquals(expected, actual) - except AssertionError as e: - raise - raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) + raise + raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) From 9ab0a423e294c0a1494481d25992a954be9917be Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 15:29:54 -0800 Subject: [PATCH 179/334] re-indent kvs.py --- keystone/backends/kvs.py | 415 ++++++++++++++++++++------------------- 1 file changed, 209 insertions(+), 206 deletions(-) diff --git a/keystone/backends/kvs.py b/keystone/backends/kvs.py index c67eac05ea..1578c6470c 100644 --- a/keystone/backends/kvs.py +++ b/keystone/backends/kvs.py @@ -1,264 +1,267 @@ -class DictKvs(dict): - def set(self, key, value): - self[key] = value +# vim: tabstop=4 shiftwidth=4 softtabstop=4 - def delete(self, key): - del self[key] + +class DictKvs(dict): + def set(self, key, value): + self[key] = value + + def delete(self, key): + del self[key] INMEMDB = DictKvs() class KvsIdentity(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db + def __init__(self, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db - # Public interface - def authenticate(self, user_id=None, tenant_id=None, password=None): - """Authenticate based on a user, tenant and password. + # Public interface + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. - Expects the user object to have a password field and the tenant to be - in the list of tenants on the user. + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. - """ - user_ref = self.get_user(user_id) - tenant_ref = None - metadata_ref = None - if not user_ref or user_ref.get('password') != password: - raise AssertionError('Invalid user / password') - if tenant_id and tenant_id not in user_ref['tenants']: - raise AssertionError('Invalid tenant') + """ + user_ref = self.get_user(user_id) + tenant_ref = None + metadata_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') + if tenant_id and tenant_id not in user_ref['tenants']: + raise AssertionError('Invalid tenant') - tenant_ref = self.get_tenant(tenant_id) - if tenant_ref: - metadata_ref = self.get_metadata(user_id, tenant_id) - else: - metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + metadata_ref = self.get_metadata(user_id, tenant_id) + else: + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) - def get_tenant(self, tenant_id): - tenant_ref = self.db.get('tenant-%s' % tenant_id) - return tenant_ref + def get_tenant(self, tenant_id): + tenant_ref = self.db.get('tenant-%s' % tenant_id) + return tenant_ref - def get_tenant_by_name(self, tenant_name): - tenant_ref = self.db.get('tenant_name-%s' % tenant_name) - return tenant_ref + def get_tenant_by_name(self, tenant_name): + tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + return tenant_ref - def get_user(self, user_id): - user_ref = self.db.get('user-%s' % user_id) - return user_ref + def get_user(self, user_id): + user_ref = self.db.get('user-%s' % user_id) + return user_ref - def get_user_by_name(self, user_name): - user_ref = self.db.get('user_name-%s' % user_name) - return user_ref + def get_user_by_name(self, user_name): + user_ref = self.db.get('user_name-%s' % user_name) + return user_ref - def get_metadata(self, user_id, tenant_id): - return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) + def get_metadata(self, user_id, tenant_id): + return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) - def get_role(self, role_id): - role_ref = self.db.get('role-%s' % role_id) - return role_ref + def get_role(self, role_id): + role_ref = self.db.get('role-%s' % role_id) + return role_ref - def list_users(self): - user_ids = self.db.get('user_list', []) - return [self.get_user(x) for x in user_ids] + def list_users(self): + user_ids = self.db.get('user_list', []) + return [self.get_user(x) for x in user_ids] - def list_roles(self): - role_ids = self.db.get('role_list', []) - return [self.get_role(x) for x in role_ids] + def list_roles(self): + role_ids = self.db.get('role_list', []) + return [self.get_role(x) for x in role_ids] # These should probably be part of the high-level API - def add_user_to_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.add(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + def add_user_to_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.add(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) - def remove_user_from_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.remove(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + def remove_user_from_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.remove(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) - def get_tenants_for_user(self, user_id): - user_ref = self.get_user(user_id) - return user_ref.get('tenants', []) + def get_tenants_for_user(self, user_id): + user_ref = self.get_user(user_id) + return user_ref.get('tenants', []) - def get_roles_for_user_and_tenant(self, user_id, tenant_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - return metadata_ref.get('roles', []) + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) - def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.add(role_id) - metadata_ref['roles'] = list(roles) - self.update_metadata(user_id, tenant_id, metadata_ref) + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.add(role_id) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) - def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.remove(role_id) - metadata_ref['roles'] = list(roles) - self.update_metadata(user_id, tenant_id, metadata_ref) + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.remove(role_id) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) - # CRUD - def create_user(self, user_id, user): - self.db.set('user-%s' % user_id, user) - self.db.set('user_name-%s' % user['name'], user) - user_list = set(self.db.get('user_list', [])) - user_list.add(user_id) - self.db.set('user_list', list(user_list)) - return user + # CRUD + def create_user(self, user_id, user): + self.db.set('user-%s' % user_id, user) + self.db.set('user_name-%s' % user['name'], user) + user_list = set(self.db.get('user_list', [])) + user_list.add(user_id) + self.db.set('user_list', list(user_list)) + return user - def update_user(self, user_id, user): - # get the old name and delete it too - old_user = self.db.get('user-%s' % user_id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % user_id, user) - self.db.set('user_name-%s' % user['name'], user) - return user + def update_user(self, user_id, user): + # get the old name and delete it too + old_user = self.db.get('user-%s' % user_id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.set('user-%s' % user_id, user) + self.db.set('user_name-%s' % user['name'], user) + return user - def delete_user(self, user_id): - old_user = self.db.get('user-%s' % user_id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.delete('user-%s' % user_id) - user_list = set(self.db.get('user_list', [])) - user_list.remove(user_id) - self.db.set('user_list', list(user_list)) - return None + def delete_user(self, user_id): + old_user = self.db.get('user-%s' % user_id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.delete('user-%s' % user_id) + user_list = set(self.db.get('user_list', [])) + user_list.remove(user_id) + self.db.set('user_list', list(user_list)) + return None - def create_tenant(self, tenant_id, tenant): - self.db.set('tenant-%s' % tenant_id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant + def create_tenant(self, tenant_id, tenant): + self.db.set('tenant-%s' % tenant_id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant - def update_tenant(self, tenant_id, tenant): - # get the old name and delete it too - old_tenant = self.db.get('tenant-%s' % tenant_id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % tenant_id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant + def update_tenant(self, tenant_id, tenant): + # get the old name and delete it too + old_tenant = self.db.get('tenant-%s' % tenant_id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.set('tenant-%s' % tenant_id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant - def delete_tenant(self, tenant_id): - old_tenant = self.db.get('tenant-%s' % tenant_id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.delete('tenant-%s' % tenant_id) - return None + def delete_tenant(self, tenant_id): + old_tenant = self.db.get('tenant-%s' % tenant_id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.delete('tenant-%s' % tenant_id) + return None - def create_metadata(self, user_id, tenant_id, metadata): - self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) - return metadata + def create_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata - def update_metadata(self, user_id, tenant_id, metadata): - self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) - return metadata + def update_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata - def delete_metadata(self, user_id, tenant_id): - self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) - return None + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) + return None - def create_role(self, role_id, role): - self.db.set('role-%s' % role_id, role) - role_list = set(self.db.get('role_list', [])) - role_list.add(role_id) - self.db.set('role_list', list(role_list)) - return role + def create_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) + role_list = set(self.db.get('role_list', [])) + role_list.add(role_id) + self.db.set('role_list', list(role_list)) + return role - def update_role(self, role_id, role): - self.db.set('role-%s' % role_id, role) - return role + def update_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) + return role - def delete_role(self, role_id): - self.db.delete('role-%s' % role_id) - role_list = set(self.db.get('role_list', [])) - role_list.remove(role_id) - self.db.set('role_list', list(role_list)) - return None + def delete_role(self, role_id): + self.db.delete('role-%s' % role_id) + role_list = set(self.db.get('role_list', [])) + role_list.remove(role_id) + self.db.set('role_list', list(role_list)) + return None class KvsToken(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db + def __init__(self, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db - # Public interface - def get_token(self, token_id): - return self.db.get('token-%s' % token_id) + # Public interface + def get_token(self, token_id): + return self.db.get('token-%s' % token_id) - def create_token(self, token_id, data): - self.db.set('token-%s' % token_id, data) - return data + def create_token(self, token_id, data): + self.db.set('token-%s' % token_id, data) + return data - def delete_token(self, token_id): - return self.db.delete('token-%s' % token_id) + def delete_token(self, token_id): + return self.db.delete('token-%s' % token_id) class KvsCatalog(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db + def __init__(self, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db - # Public interface - def get_catalog(self, user_id, tenant_id, metadata=None): - return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) + # Public interface + def get_catalog(self, user_id, tenant_id, metadata=None): + return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) - def get_service(self, service_id): - return self.db.get('service-%s' % service_id) + def get_service(self, service_id): + return self.db.get('service-%s' % service_id) - def list_services(self): - return self.db.get('service_list', []) + def list_services(self): + return self.db.get('service_list', []) - def create_service(self, service_id, service): - self.db.set('service-%s' % service_id, service) - service_list = set(self.db.get('service_list', [])) - service_list.add(service_id) - self.db.set('service_list', list(service_list)) - return service + def create_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) + service_list = set(self.db.get('service_list', [])) + service_list.add(service_id) + self.db.set('service_list', list(service_list)) + return service - def update_service(self, service_id, service): - self.db.set('service-%s' % service_id, service) - return service + def update_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) + return service - def delete_service(self, service_id): - self.db.delete('service-%s' % service_id) - service_list = set(self.db.get('service_list', [])) - service_list.remove(service_id) - self.db.set('service_list', list(service_list)) - return None + def delete_service(self, service_id): + self.db.delete('service-%s' % service_id) + service_list = set(self.db.get('service_list', [])) + service_list.remove(service_id) + self.db.set('service_list', list(service_list)) + return None - # Private interface - def _create_catalog(self, user_id, tenant_id, data): - self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) - return data + # Private interface + def _create_catalog(self, user_id, tenant_id, data): + self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) + return data class KvsPolicy(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db + def __init__(self, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db - def can_haz(self, target, action, credentials): - pass + def can_haz(self, target, action, credentials): + pass From bd974c9b514a539dc15ec8aa1d887ce74248c40f Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 15:58:06 -0800 Subject: [PATCH 180/334] re-indent --- keystone/backends/policy.py | 25 +- keystone/backends/sql/core.py | 712 +++++++++++++++++----------------- 2 files changed, 372 insertions(+), 365 deletions(-) diff --git a/keystone/backends/policy.py b/keystone/backends/policy.py index bb33548db1..643d3e2136 100644 --- a/keystone/backends/policy.py +++ b/keystone/backends/policy.py @@ -1,18 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + + import logging class TrivialTrue(object): - def can_haz(self, target, credentials): - return True + def can_haz(self, target, credentials): + return True class SimpleMatch(object): - def can_haz(self, target, credentials): - """Check whether key-values in target are present in credentials.""" - # TODO(termie): handle ANDs, probably by providing a tuple instead of a - # string - for requirement in target: - key, match = requirement.split(':', 1) - check = credentials.get(key) - if check == match: - return True + def can_haz(self, target, credentials): + """Check whether key-values in target are present in credentials.""" + # TODO(termie): handle ANDs, probably by providing a tuple instead of a + # string + for requirement in target: + key, match = requirement.split(':', 1) + check = credentials.get(key) + if check == match: + return True diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index b75025313d..bfc3a97300 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -1,5 +1,8 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + """SQL backends for the various services.""" + import json import eventlet.db_pool @@ -22,441 +25,442 @@ Base = declarative.declarative_base() # Special Fields class JsonBlob(sql_types.TypeDecorator): - impl = sql.Text + impl = sql.Text - def process_bind_param(self, value, dialect): - return json.dumps(value) + def process_bind_param(self, value, dialect): + return json.dumps(value) - def process_result_value(self, value, dialect): - return json.loads(value) + def process_result_value(self, value, dialect): + return json.loads(value) class DictBase(object): - def to_dict(self): - return dict(self.iteritems()) + def to_dict(self): + return dict(self.iteritems()) - def __setitem__(self, key, value): - setattr(self, key, value) + def __setitem__(self, key, value): + setattr(self, key, value) - def __getitem__(self, key): - return getattr(self, key) + def __getitem__(self, key): + return getattr(self, key) - def get(self, key, default=None): - return getattr(self, key, default) + def get(self, key, default=None): + return getattr(self, key, default) - def __iter__(self): - self._i = iter(sqlalchemy.orm.object_mapper(self).columns) - return self + def __iter__(self): + self._i = iter(sqlalchemy.orm.object_mapper(self).columns) + return self - def next(self): - n = self._i.next().name - return n + def next(self): + n = self._i.next().name + return n - def update(self, values): - """Make the model object behave like a dict.""" - for k, v in values.iteritems(): - setattr(self, k, v) + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in values.iteritems(): + setattr(self, k, v) - def iteritems(self): - """Make the model object behave like a dict. + def iteritems(self): + """Make the model object behave like a dict. - Includes attributes from joins. + Includes attributes from joins. - """ - return dict([(k, getattr(self, k)) for k in self]) - #local = dict(self) - #joined = dict([(k, v) for k, v in self.__dict__.iteritems() - # if not k[0] == '_']) - #local.update(joined) - #return local.iteritems() + """ + return dict([(k, getattr(self, k)) for k in self]) + #local = dict(self) + #joined = dict([(k, v) for k, v in self.__dict__.iteritems() + # if not k[0] == '_']) + #local.update(joined) + #return local.iteritems() # Tables class User(Base, DictBase): - __tablename__ = 'user' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - #password = sql.Column(sql.String(64)) - extra = sql.Column(JsonBlob()) + __tablename__ = 'user' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + #password = sql.Column(sql.String(64)) + extra = sql.Column(JsonBlob()) - @classmethod - def from_dict(cls, user_dict): - # shove any non-indexed properties into extra - extra = {} - for k, v in user_dict.copy().iteritems(): - # TODO(termie): infer this somehow - if k not in ['id', 'name']: - extra[k] = user_dict.pop(k) + @classmethod + def from_dict(cls, user_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in user_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = user_dict.pop(k) - user_dict['extra'] = extra - return cls(**user_dict) + user_dict['extra'] = extra + return cls(**user_dict) - def to_dict(self): - extra_copy = self.extra.copy() - extra_copy['id'] = self.id - extra_copy['name'] = self.name - return extra_copy + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy class Tenant(Base, DictBase): - __tablename__ = 'tenant' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - extra = sql.Column(JsonBlob()) + __tablename__ = 'tenant' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + extra = sql.Column(JsonBlob()) - @classmethod - def from_dict(cls, tenant_dict): - # shove any non-indexed properties into extra - extra = {} - for k, v in tenant_dict.copy().iteritems(): - # TODO(termie): infer this somehow - if k not in ['id', 'name']: - extra[k] = tenant_dict.pop(k) + @classmethod + def from_dict(cls, tenant_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in tenant_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = tenant_dict.pop(k) - tenant_dict['extra'] = extra - return cls(**tenant_dict) + tenant_dict['extra'] = extra + return cls(**tenant_dict) - def to_dict(self): - extra_copy = self.extra.copy() - extra_copy['id'] = self.id - extra_copy['name'] = self.name - return extra_copy + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy class Role(Base, DictBase): - __tablename__ = 'role' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64)) + __tablename__ = 'role' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64)) class Metadata(Base, DictBase): - __tablename__ = 'metadata' - #__table_args__ = ( - # sql.Index('idx_metadata_usertenant', 'user', 'tenant'), - # ) + __tablename__ = 'metadata' + #__table_args__ = ( + # sql.Index('idx_metadata_usertenant', 'user', 'tenant'), + # ) - user_id = sql.Column(sql.String(64), primary_key=True) - tenant_id = sql.Column(sql.String(64), primary_key=True) - data = sql.Column(JsonBlob()) + user_id = sql.Column(sql.String(64), primary_key=True) + tenant_id = sql.Column(sql.String(64), primary_key=True) + data = sql.Column(JsonBlob()) class Token(Base, DictBase): - __tablename__ = 'token' - id = sql.Column(sql.String(64), primary_key=True) - user = sql.Column(sql.String(64)) - tenant = sql.Column(sql.String(64)) - data = sql.Column(JsonBlob()) + __tablename__ = 'token' + id = sql.Column(sql.String(64), primary_key=True) + user = sql.Column(sql.String(64)) + tenant = sql.Column(sql.String(64)) + data = sql.Column(JsonBlob()) class UserTenantMembership(Base, DictBase): - """Tenant membership join table.""" - __tablename__ = 'user_tenant_membership' - user_id = sql.Column(sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True) - tenant_id = sql.Column(sql.String(64), - sql.ForeignKey('tenant.id'), + """Tenant membership join table.""" + __tablename__ = 'user_tenant_membership' + user_id = sql.Column(sql.String(64), + sql.ForeignKey('user.id'), primary_key=True) + tenant_id = sql.Column(sql.String(64), + sql.ForeignKey('tenant.id'), + primary_key=True) # Backends class SqlBase(object): - _MAKER = None - _ENGINE = None + _MAKER = None + _ENGINE = None - def get_session(self, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy session.""" - if self._MAKER is None or self._ENGINE is None: - self._ENGINE = self.get_engine() - self._MAKER = self.get_maker(self._ENGINE, autocommit, expire_on_commit) + def get_session(self, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy session.""" + if self._MAKER is None or self._ENGINE is None: + self._ENGINE = self.get_engine() + self._MAKER = self.get_maker(self._ENGINE, + autocommit, + expire_on_commit) - session = self._MAKER() - # TODO(termie): we may want to do something similar - #session.query = nova.exception.wrap_db_error(session.query) - #session.flush = nova.exception.wrap_db_error(session.flush) - return session + session = self._MAKER() + # TODO(termie): we may want to do something similar + #session.query = nova.exception.wrap_db_error(session.query) + #session.flush = nova.exception.wrap_db_error(session.flush) + return session - def get_engine(self): - """Return a SQLAlchemy engine.""" - connection_dict = sqlalchemy.engine.url.make_url(CONF.sql.connection) + def get_engine(self): + """Return a SQLAlchemy engine.""" + connection_dict = sqlalchemy.engine.url.make_url(CONF.sql.connection) - engine_args = { - "pool_recycle": CONF.sql.idle_timeout, - "echo": True, - } + engine_args = {"pool_recycle": CONF.sql.idle_timeout, + "echo": True, + } - if "sqlite" in connection_dict.drivername: - engine_args["poolclass"] = sqlalchemy.pool.NullPool + if "sqlite" in connection_dict.drivername: + engine_args["poolclass"] = sqlalchemy.pool.NullPool - return sql.create_engine(CONF.sql.connection, **engine_args) + return sql.create_engine(CONF.sql.connection, **engine_args) - def get_maker(self, engine, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy sessionmaker using the given engine.""" - return sqlalchemy.orm.sessionmaker(bind=engine, - autocommit=autocommit, - expire_on_commit=expire_on_commit) + def get_maker(self, engine, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy sessionmaker using the given engine.""" + return sqlalchemy.orm.sessionmaker(bind=engine, + autocommit=autocommit, + expire_on_commit=expire_on_commit) class SqlIdentity(SqlBase): - # Internal interface to manage the database - def db_sync(self): - migration.db_sync() + # Internal interface to manage the database + def db_sync(self): + migration.db_sync() # Identity interface - def authenticate(self, user_id=None, tenant_id=None, password=None): - """Authenticate based on a user, tenant and password. + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. - Expects the user object to have a password field and the tenant to be - in the list of tenants on the user. + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. - """ - user_ref = self.get_user(user_id) - tenant_ref = None - metadata_ref = None - if not user_ref or user_ref.get('password') != password: - raise AssertionError('Invalid user / password') + """ + user_ref = self.get_user(user_id) + tenant_ref = None + metadata_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') - tenants = self.get_tenants_for_user(user_id) - if tenant_id and tenant_id not in tenants: - raise AssertionError('Invalid tenant') + tenants = self.get_tenants_for_user(user_id) + if tenant_id and tenant_id not in tenants: + raise AssertionError('Invalid tenant') - tenant_ref = self.get_tenant(tenant_id) - print 'ETESTSET', tenant_ref - if tenant_ref: - metadata_ref = self.get_metadata(user_id, tenant_id) - else: - metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + metadata_ref = self.get_metadata(user_id, tenant_id) + else: + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) - def get_tenant(self, tenant_id): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - if not tenant_ref: - return - return tenant_ref.to_dict() + def get_tenant(self, tenant_id): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + if not tenant_ref: + return + return tenant_ref.to_dict() - def get_tenant_by_name(self, tenant_name): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() - if not tenant_ref: - return - return tenant_ref.to_dict() + def get_tenant_by_name(self, tenant_name): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() + if not tenant_ref: + return + return tenant_ref.to_dict() - def get_user(self, user_id): - session = self.get_session() - user_ref = session.query(User).filter_by(id=user_id).first() - if not user_ref: - return - return user_ref.to_dict() + def get_user(self, user_id): + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + if not user_ref: + return + return user_ref.to_dict() - def get_user_by_name(self, user_name): - session = self.get_session() - user_ref = session.query(User).filter_by(name=user_name).first() - if not user_ref: - return - return user_ref.to_dict() + def get_user_by_name(self, user_name): + session = self.get_session() + user_ref = session.query(User).filter_by(name=user_name).first() + if not user_ref: + return + return user_ref.to_dict() - def get_metadata(self, user_id, tenant_id): - session = self.get_session() - metadata_ref = session.query(Metadata)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - return getattr(metadata_ref, 'data', None) + def get_metadata(self, user_id, tenant_id): + session = self.get_session() + metadata_ref = session.query(Metadata)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + return getattr(metadata_ref, 'data', None) - def get_role(self, role_id): - session = self.get_session() - role_ref = session.query(Role).filter_by(id=role_id).first() - return role_ref + def get_role(self, role_id): + session = self.get_session() + role_ref = session.query(Role).filter_by(id=role_id).first() + return role_ref - def list_users(self): - session = self.get_session() - user_refs = session.query(User) - return [x.to_dict() for x in user_refs] + def list_users(self): + session = self.get_session() + user_refs = session.query(User) + return [x.to_dict() for x in user_refs] - def list_roles(self): - session = self.get_session() - role_refs = session.query(Role) - return list(role_refs) + def list_roles(self): + session = self.get_session() + role_refs = session.query(Role) + return list(role_refs) - # These should probably be part of the high-level API - def add_user_to_tenant(self, tenant_id, user_id): - session = self.get_session() - q = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id) - rv = q.first() - if rv: - return + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + session = self.get_session() + q = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id) + rv = q.first() + if rv: + return - with session.begin(): - session.add(UserTenantMembership(user_id=user_id, tenant_id=tenant_id)) - session.flush() + with session.begin(): + session.add(UserTenantMembership(user_id=user_id, + tenant_id=tenant_id)) + session.flush() - def remove_user_from_tenant(self, tenant_id, user_id): - session = self.get_session() - membership_ref = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - with session.begin(): - session.delete(membership_ref) - session.flush() + def remove_user_from_tenant(self, tenant_id, user_id): + session = self.get_session() + membership_ref = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + with session.begin(): + session.delete(membership_ref) + session.flush() - def get_tenants_for_user(self, user_id): - session = self.get_session() - membership_refs = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .all() - - return [x.tenant_id for x in membership_refs] - - def get_roles_for_user_and_tenant(self, user_id, tenant_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - return metadata_ref.get('roles', []) - - def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - is_new = False - if not metadata_ref: - is_new = True - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.add(role_id) - metadata_ref['roles'] = list(roles) - if not is_new: - self.update_metadata(user_id, tenant_id, metadata_ref) - else: - self.create_metadata(user_id, tenant_id, metadata_ref) - - def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - is_new = False - if not metadata_ref: - is_new = True - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.remove(role_id) - metadata_ref['roles'] = list(roles) - if not is_new: - self.update_metadata(user_id, tenant_id, metadata_ref) - else: - self.create_metadata(user_id, tenant_id, metadata_ref) - - # CRUD - def create_user(self, user_id, user): - session = self.get_session() - with session.begin(): - user_ref = User.from_dict(user) - session.add(user_ref) - session.flush() - return user_ref.to_dict() - - def update_user(self, user_id, user): - session = self.get_session() - with session.begin(): - user_ref = session.query(User).filter_by(id=user_id).first() - old_user_dict = user_ref.to_dict() - for k in user: - old_user_dict[k] = user[k] - new_user = User.from_dict(old_user_dict) - - user_ref.name = new_user.name - user_ref.extra = new_user.extra - session.flush() - return user_ref - - def delete_user(self, user_id): - session = self.get_session() - user_ref = session.query(User).filter_by(id=user_id).first() - with session.begin(): - session.delete(user_ref) - session.flush() - - def create_tenant(self, tenant_id, tenant): - session = self.get_session() - with session.begin(): - tenant_ref = Tenant.from_dict(tenant) - session.add(tenant_ref) - session.flush() - return tenant_ref.to_dict() - - def update_tenant(self, tenant_id, tenant): - session = self.get_session() - with session.begin(): - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - old_tenant_dict = tenant_ref.to_dict() - for k in tenant: - old_tenant_dict[k] = tenant[k] - new_tenant = Tenant.from_dict(old_tenant_dict) - - tenant_ref.name = new_tenant.name - tenant_ref.extra = new_tenant.extra - session.flush() - return tenant_ref - - def delete_tenant(self, tenant_id): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - with session.begin(): - session.delete(tenant_ref) - session.flush() - - def create_metadata(self, user_id, tenant_id, metadata): - session = self.get_session() - with session.begin(): - session.add(Metadata(user_id=user_id, - tenant_id=tenant_id, - data=metadata)) - session.flush() - return metadata - - def update_metadata(self, user_id, tenant_id, metadata): - session = self.get_session() - with session.begin(): - metadata_ref = session.query(Metadata)\ + def get_tenants_for_user(self, user_id): + session = self.get_session() + membership_refs = session.query(UserTenantMembership)\ .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - data = metadata_ref.data.copy() - for k in metadata: - data[k] = metadata[k] - metadata_ref.data = data - session.flush() - return metadata_ref + .all() - def delete_metadata(self, user_id, tenant_id): - self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) - return None + return [x.tenant_id for x in membership_refs] - def create_role(self, role_id, role): - session = self.get_session() - with session.begin(): - session.add(Role(**role)) - session.flush() - return role + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) - def update_role(self, role_id, role): - session = self.get_session() - with session.begin(): - role_ref = session.query(Role).filter_by(id=role_id).first() - for k in role: - role_ref[k] = role[k] - session.flush() - return role_ref + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + is_new = False + if not metadata_ref: + is_new = True + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.add(role_id) + metadata_ref['roles'] = list(roles) + if not is_new: + self.update_metadata(user_id, tenant_id, metadata_ref) + else: + self.create_metadata(user_id, tenant_id, metadata_ref) - def delete_role(self, role_id): - session = self.get_session() - role_ref = session.query(Role).filter_by(id=role_id).first() - with session.begin(): - session.delete(role_ref) + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + is_new = False + if not metadata_ref: + is_new = True + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.remove(role_id) + metadata_ref['roles'] = list(roles) + if not is_new: + self.update_metadata(user_id, tenant_id, metadata_ref) + else: + self.create_metadata(user_id, tenant_id, metadata_ref) + + # CRUD + def create_user(self, user_id, user): + session = self.get_session() + with session.begin(): + user_ref = User.from_dict(user) + session.add(user_ref) + session.flush() + return user_ref.to_dict() + + def update_user(self, user_id, user): + session = self.get_session() + with session.begin(): + user_ref = session.query(User).filter_by(id=user_id).first() + old_user_dict = user_ref.to_dict() + for k in user: + old_user_dict[k] = user[k] + new_user = User.from_dict(old_user_dict) + + user_ref.name = new_user.name + user_ref.extra = new_user.extra + session.flush() + return user_ref + + def delete_user(self, user_id): + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + with session.begin(): + session.delete(user_ref) + session.flush() + + def create_tenant(self, tenant_id, tenant): + session = self.get_session() + with session.begin(): + tenant_ref = Tenant.from_dict(tenant) + session.add(tenant_ref) + session.flush() + return tenant_ref.to_dict() + + def update_tenant(self, tenant_id, tenant): + session = self.get_session() + with session.begin(): + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + old_tenant_dict = tenant_ref.to_dict() + for k in tenant: + old_tenant_dict[k] = tenant[k] + new_tenant = Tenant.from_dict(old_tenant_dict) + + tenant_ref.name = new_tenant.name + tenant_ref.extra = new_tenant.extra + session.flush() + return tenant_ref + + def delete_tenant(self, tenant_id): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + with session.begin(): + session.delete(tenant_ref) + session.flush() + + def create_metadata(self, user_id, tenant_id, metadata): + session = self.get_session() + with session.begin(): + session.add(Metadata(user_id=user_id, + tenant_id=tenant_id, + data=metadata)) + session.flush() + return metadata + + def update_metadata(self, user_id, tenant_id, metadata): + session = self.get_session() + with session.begin(): + metadata_ref = session.query(Metadata)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + data = metadata_ref.data.copy() + for k in metadata: + data[k] = metadata[k] + metadata_ref.data = data + session.flush() + return metadata_ref + + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) + return None + + def create_role(self, role_id, role): + session = self.get_session() + with session.begin(): + session.add(Role(**role)) + session.flush() + return role + + def update_role(self, role_id, role): + session = self.get_session() + with session.begin(): + role_ref = session.query(Role).filter_by(id=role_id).first() + for k in role: + role_ref[k] = role[k] + session.flush() + return role_ref + + def delete_role(self, role_id): + session = self.get_session() + role_ref = session.query(Role).filter_by(id=role_id).first() + with session.begin(): + session.delete(role_ref) class SqlToken(SqlBase): - pass + pass class SqlCatalog(SqlBase): - pass + pass From c233dc215cab776d3e7faf10cf1870fa84d24db5 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 16:02:37 -0800 Subject: [PATCH 181/334] re-indent --- keystone/backends/sql/util.py | 14 ++-- keystone/backends/templated.py | 117 +++++++++++++++++---------------- 2 files changed, 67 insertions(+), 64 deletions(-) diff --git a/keystone/backends/sql/util.py b/keystone/backends/sql/util.py index 498ee6c2a0..e96ac6c631 100644 --- a/keystone/backends/sql/util.py +++ b/keystone/backends/sql/util.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + import os from keystone import config @@ -8,9 +10,9 @@ CONF = config.CONF def setup_test_database(): - # TODO(termie): be smart about this - try: - os.unlink('bla.db') - except Exception: - pass - migration.db_sync() + # TODO(termie): be smart about this + try: + os.unlink('bla.db') + except Exception: + pass + migration.db_sync() diff --git a/keystone/backends/templated.py b/keystone/backends/templated.py index 989ee0f679..475f13daf7 100644 --- a/keystone/backends/templated.py +++ b/keystone/backends/templated.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + from keystone import config from keystone import logging from keystone.backends import kvs @@ -8,80 +10,79 @@ config.register_str('template_file', group='catalog') class TemplatedCatalog(kvs.KvsCatalog): - """A backend that generates endpoints for the Catalog based on templates. + """A backend that generates endpoints for the Catalog based on templates. - It is usually configured via config entries that look like: + It is usually configured via config entries that look like: - catalog.$REGION.$SERVICE.$key = $value + catalog.$REGION.$SERVICE.$key = $value - and is stored in a similar looking hierarchy. Where a value can contain - values to be interpolated by standard python string interpolation that look - like (the % is replaced by a $ due to paste attmepting to interpolate on its - own: + and is stored in a similar looking hierarchy. Where a value can contain + values to be interpolated by standard python string interpolation that look + like (the % is replaced by a $ due to paste attmepting to interpolate on + its own: - http://localhost:$(public_port)s/ + http://localhost:$(public_port)s/ - When expanding the template it will pass in a dict made up of the conf - instance plus a few additional key-values, notably tenant_id and user_id. + When expanding the template it will pass in a dict made up of the conf + instance plus a few additional key-values, notably tenant_id and user_id. - It does not care what the keys and values are but it is worth noting that - keystone_compat will expect certain keys to be there so that it can munge - them into the output format keystone expects. These keys are: + It does not care what the keys and values are but it is worth noting that + keystone_compat will expect certain keys to be there so that it can munge + them into the output format keystone expects. These keys are: - name - the name of the service, most likely repeated for all services of - the same type, across regions. - adminURL - the url of the admin endpoint - publicURL - the url of the public endpoint - internalURL - the url of the internal endpoint + name - the name of the service, most likely repeated for all services of + the same type, across regions. + adminURL - the url of the admin endpoint + publicURL - the url of the public endpoint + internalURL - the url of the internal endpoint - """ + """ - def __init__(self, templates=None): - if templates: - self.templates = templates - else: - self._load_templates(CONF.catalog.template_file) + def __init__(self, templates=None): + if templates: + self.templates = templates + else: + self._load_templates(CONF.catalog.template_file) + super(TemplatedCatalog, self).__init__() - super(TemplatedCatalog, self).__init__() + def _load_templates(self, template_file): + o = {} + for line in open(template_file): + if ' = ' not in line: + continue - def _load_templates(self, template_file): - o = {} - for line in open(template_file): - if ' = ' not in line: - continue + k, v = line.strip().split(' = ') + if not k.startswith('catalog.'): + continue - k, v = line.strip().split(' = ') - if not k.startswith('catalog.'): - continue + parts = k.split('.') - parts = k.split('.') + region = parts[1] + # NOTE(termie): object-store insists on having a dash + service = parts[2].replace('_', '-') + key = parts[3] - region = parts[1] - # NOTE(termie): object-store insists on having a dash - service = parts[2].replace('_', '-') - key = parts[3] + region_ref = o.get(region, {}) + service_ref = region_ref.get(service, {}) + service_ref[key] = v - region_ref = o.get(region, {}) - service_ref = region_ref.get(service, {}) - service_ref[key] = v + region_ref[service] = service_ref + o[region] = region_ref - region_ref[service] = service_ref - o[region] = region_ref + self.templates = o - self.templates = o + def get_catalog(self, user_id, tenant_id, metadata=None): + d = dict(CONF.iteritems()) + d.update({'tenant_id': tenant_id, + 'user_id': user_id}) - def get_catalog(self, user_id, tenant_id, metadata=None): - d = dict(CONF.iteritems()) - d.update({'tenant_id': tenant_id, - 'user_id': user_id}) + o = {} + for region, region_ref in self.templates.iteritems(): + o[region] = {} + for service, service_ref in region_ref.iteritems(): + o[region][service] = {} + for k, v in service_ref.iteritems(): + v = v.replace('$(', '%(') + o[region][service][k] = v % d - o = {} - for region, region_ref in self.templates.iteritems(): - o[region] = {} - for service, service_ref in region_ref.iteritems(): - o[region][service] = {} - for k, v in service_ref.iteritems(): - v = v.replace('$(', '%(') - o[region][service][k] = v % d - - return o + return o From 8c33e66f1d1fe4a439b27a52584437174d14cd60 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 12 Jan 2012 16:07:23 -0800 Subject: [PATCH 182/334] re-indent --- keystone/middleware/internal.py | 106 ++++++++++++++++---------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/keystone/middleware/internal.py b/keystone/middleware/internal.py index 5bac8fb27b..ccf68b82d1 100644 --- a/keystone/middleware/internal.py +++ b/keystone/middleware/internal.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + import json from keystone import config @@ -20,79 +22,79 @@ PARAMS_ENV = 'openstack.params' class TokenAuthMiddleware(wsgi.Middleware): - def process_request(self, request): - token = request.headers.get(AUTH_TOKEN_HEADER) - context = request.environ.get(CONTEXT_ENV, {}) - context['token_id'] = token - request.environ[CONTEXT_ENV] = context + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['token_id'] = token + request.environ[CONTEXT_ENV] = context class AdminTokenAuthMiddleware(wsgi.Middleware): - """A trivial filter that checks for a pre-defined admin token. + """A trivial filter that checks for a pre-defined admin token. - Sets 'is_admin' to true in the context, expected to be checked by - methods that are admin-only. + Sets 'is_admin' to true in the context, expected to be checked by + methods that are admin-only. - """ + """ - def process_request(self, request): - token = request.headers.get(AUTH_TOKEN_HEADER) - context = request.environ.get(CONTEXT_ENV, {}) - context['is_admin'] = (token == CONF.admin_token) - request.environ[CONTEXT_ENV] = context + def process_request(self, request): + token = request.headers.get(AUTH_TOKEN_HEADER) + context = request.environ.get(CONTEXT_ENV, {}) + context['is_admin'] = (token == CONF.admin_token) + request.environ[CONTEXT_ENV] = context class PostParamsMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as POST parameters. + """Middleware to allow method arguments to be passed as POST parameters. - Filters out the parameters `self`, `context` and anything beginning with - an underscore. + Filters out the parameters `self`, `context` and anything beginning with + an underscore. - """ + """ - def process_request(self, request): - params_parsed = request.params - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v - request.environ[PARAMS_ENV] = params + request.environ[PARAMS_ENV] = params class JsonBodyMiddleware(wsgi.Middleware): - """Middleware to allow method arguments to be passed as serialized JSON. + """Middleware to allow method arguments to be passed as serialized JSON. - Accepting arguments as JSON is useful for accepting data that may be more - complex than simple primitives. + Accepting arguments as JSON is useful for accepting data that may be more + complex than simple primitives. - In this case we accept it as urlencoded data under the key 'json' as in - json= but this could be extended to accept raw JSON - in the POST body. + In this case we accept it as urlencoded data under the key 'json' as in + json= but this could be extended to accept raw JSON + in the POST body. - Filters out the parameters `self`, `context` and anything beginning with - an underscore. + Filters out the parameters `self`, `context` and anything beginning with + an underscore. - """ + """ - def process_request(self, request): - #if 'json' not in request.params: - # return + def process_request(self, request): + #if 'json' not in request.params: + # return - params_json = request.body - if not params_json: - return + params_json = request.body + if not params_json: + return - params_parsed = json.loads(params_json) - params = {} - for k, v in params_parsed.iteritems(): - if k in ('self', 'context'): - continue - if k.startswith('_'): - continue - params[k] = v + params_parsed = json.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v - request.environ[PARAMS_ENV] = params + request.environ[PARAMS_ENV] = params From 2ed9759ff729163113d42109c846ec19374c9c26 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 16 Jan 2012 14:59:32 -0800 Subject: [PATCH 183/334] update readme --- README.rst | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 2c9e5534ce..46c3ec6004 100644 --- a/README.rst +++ b/README.rst @@ -112,29 +112,6 @@ interpolation):: catalog.RegionOne.identity.name = 'Identity Service' ---------------- -Keystone Compat ---------------- - -While Keystone Light takes a fundamentally different approach to its services -from Keystone, a compatibility layer is included to make switching much easier -for projects already attempting to use Keystone. - -The compatibility service is activated by defining an alternate application in -the paste.deploy config and adding it to your main pipeline:: - - [app:keystone] - paste.app_factory = keystonelight.keystone_compat:app_factory - -Also relevant to Keystone compatibility are these sequence diagrams (openable -with sdedit_) - -.. _sdedit: http://sourceforge.net/projects/sdedit/files/sdedit/4.0/ - -Diagram: keystone_compat_flows.sdx_ - -.. _: https://raw.github.com/termie/keystonelight/master/docs/keystone_compat_flows.sdx - ---------------- Approach to CRUD ---------------- @@ -220,11 +197,5 @@ then do what is effectively a 'Simple Match' style match against the creds. Still To Do ----------- - * Dev and testing setups would do well with some user/tenant/etc CRUD, for the - KVS backends at least. - * Fixture loading functionality would also be killer tests and dev. * LDAP backend. - * Keystone import. - * (./) Admin-only interface - * Don't check git checkouts as often, to speed up tests - * common config - http://wiki.openstack.org/CommonConfigModule + * Diablo migration. From afd897f9122cdee925376a1c25994a515082963f Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 16 Jan 2012 16:31:44 -0800 Subject: [PATCH 184/334] add an ec2 extension --- etc/keystone.conf | 6 +- keystone/backends/kvs.py | 36 +++++++++- keystone/config.py | 1 + keystone/ec2.py | 12 ++++ keystone/service.py | 137 +++++++++++++++++++++++++++++++++++++-- keystone/utils.py | 81 +++++++++++++++++++++++ 6 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 keystone/ec2.py diff --git a/etc/keystone.conf b/etc/keystone.conf index 517cfd43a5..b937be45b5 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -35,6 +35,8 @@ driver = keystone.backends.kvs.KvsToken [policy] driver = keystone.backends.policy.SimpleMatch +[ec2] +driver = keystone.backends.kvs.KvsEc2 [filter:debug] paste.filter_factory = keystone.wsgi:Debug.factory @@ -51,6 +53,8 @@ paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] paste.filter_factory = keystone.service:AdminCrudExtension.factory +[filter:ec2_extension] +paste.filter_factory = keystone.service:Ec2Extension.factory [app:public_service] paste.app_factory = keystone.service:public_app_factory @@ -59,7 +63,7 @@ paste.app_factory = keystone.service:public_app_factory paste.app_factory = keystone.service:admin_app_factory [pipeline:public_api] -pipeline = token_auth admin_token_auth json_body debug public_service +pipeline = token_auth admin_token_auth json_body debug ec2_extension public_service [pipeline:admin_api] pipeline = token_auth admin_token_auth json_body debug crud_extension admin_service diff --git a/keystone/backends/kvs.py b/keystone/backends/kvs.py index 1578c6470c..3ab5f1e9a4 100644 --- a/keystone/backends/kvs.py +++ b/keystone/backends/kvs.py @@ -74,7 +74,7 @@ class KvsIdentity(object): role_ids = self.db.get('role_list', []) return [self.get_role(x) for x in role_ids] - # These should probably be part of the high-level API + # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): user_ref = self.get_user(user_id) tenants = set(user_ref.get('tenants', [])) @@ -265,3 +265,37 @@ class KvsPolicy(object): def can_haz(self, target, action, credentials): pass + + +class KvsEc2(object): + def __init__(self, db=None): + if db is None: + db = INMEMDB + elif type(db) is type({}): + db = DictKvs(db) + self.db = db + + # Public interface + def get_credential(self, credential_id): + credential_ref = self.db.get('credential-%s' % credential_id) + return credential_ref + + def list_credentials(self): + credential_ids = self.db.get('credential_list', []) + return [self.get_credential(x) for x in credential_ids] + + # CRUD + def create_credential(self, credential_id, credential): + self.db.set('credential-%s' % credential_id, credential) + credential_list = set(self.db.get('credential_list', [])) + credential_list.add(credential_id) + self.db.set('credential_list', list(credential_list)) + return credential + + def delete_credential(self, credential_id): + old_credential = self.db.get('credential-%s' % credential_id) + self.db.delete('credential-%s' % credential_id) + credential_list = set(self.db.get('credential_list', [])) + credential_list.remove(credential_id) + self.db.set('credential_list', list(credential_list)) + return None diff --git a/keystone/config.py b/keystone/config.py index 6cbc1571b8..bc5a2683ce 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -123,3 +123,4 @@ register_str('driver', group='catalog') register_str('driver', group='identity') register_str('driver', group='policy') register_str('driver', group='token') +register_str('driver', group='ec2') diff --git a/keystone/ec2.py b/keystone/ec2.py new file mode 100644 index 0000000000..a0f1c0b30e --- /dev/null +++ b/keystone/ec2.py @@ -0,0 +1,12 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone import config +from keystone import manager + + +CONF = config.CONF + + +class Manager(manager.Manager): + def __init__(self): + super(Manager, self).__init__(CONF.ec2.driver) diff --git a/keystone/service.py b/keystone/service.py index b66fafb9e8..fe466d58e1 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -10,6 +10,7 @@ import webob.dec import webob.exc from keystone import catalog +from keystone import ec2 from keystone import identity from keystone import logging from keystone import policy @@ -151,10 +152,6 @@ class PublicRouter(wsgi.Router): controller=auth_controller, action='authenticate', conditions=dict(method=['POST'])) - mapper.connect('/ec2tokens', - controller=auth_controller, - action='authenticate_ec2', - conditions=dict(methods=['POST'])) # Tenant Operations tenant_controller = TenantController() @@ -329,6 +326,135 @@ class AdminCrudExtension(wsgi.ExtensionRouter): application, mapper) +class Ec2Extension(wsgi.ExtensionRouter): + def __init__(self, application): + mapper = routes.Mapper() + ec2_controller = Ec2Controller() + + # validation + mapper.connect('/ec2tokens', + controller=ec2_controller, + action='authenticate_ec2', + conditions=dict(methods=['POST'])) + + # crud + mapper.connect('/users/{user_id}/credentials/OS-EC2', + controller=ec2_controller, + action='create_credential', + conditions=dict(methods=['POST'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2', + controller=ec2_controller, + action='get_credentials', + conditions=dict(methods=['GET'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', + controller=ec2_controller, + action='get_credential', + conditions=dict(methods=['GET'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', + controller=ec2_controller, + action='delete_credential', + conditions=dict(methods=['DELETE'])) + + super(Ec2Extension, self).__init__(application, mapper) + + +class Ec2Controller(Application): + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() + self.ec2_api = ec2.Manager() + super(Ec2Controller, self).__init__() + + def authenticate_ec2(self, context, credentials=None, + ec2Credentials=None): + """Validate a signed EC2 request and provide a token.""" + # NOTE(termie): backwards compat hack + if not ecredentials and ec2Credentials: + credentials = ec2Credentials + creds_ref = self.ec2_api.get_credential(context, + credentials['access']) + + signer = utils.Signer(creds_ref['secret']) + signature = signer.generate(credentials) + if signature == credentials['signature']: + pass + # NOTE(vish): Some libraries don't use the port when signing + # requests, so try again without port. + elif ':' in credentials['signature']: + hostname, _port = credentials['host'].split(":") + credentials['host'] = hostname + signature = signer.generate(credentials) + if signature != credentials.signature: + # TODO(termie): proper exception + raise Exception("Not Authorized") + else: + raise Exception("Not Authorized") + + # TODO(termie): don't create new tokens every time + # TODO(termie): this is copied from TokenController.authenticate + token_id = uuid.uuid4().hex + tenant_ref = self.identity_api.get_tenant(creds_ref['tenant_id']) + user_ref = self.identity_api.get_user(creds_ref['user_id']) + metadata_ref = self.identity_api.get_metadata( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id']) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + metadata=metadata_ref) + + token_ref = self.token_api.create_token( + context, token_id, dict(expires='', + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) + + # TODO(termie): optimize this call at some point and put it into the + # the return for metadata + # fill out the roles in the metadata + roles_ref = [] + for role_id in metadata_ref.get('roles', []): + roles_ref.append(self.identity_api.get_role(context, role_id)) + + # TODO(termie): make this a util function or something + # TODO(termie): i don't think the ec2 middleware currently expects a + # full return, but it contains a note saying that it + # would be better to expect a full return + return TokenController._format_authenticate( + self, token_ref, roles_ref, catalog_ref) + + def create_credential(self, context, user_id, tenant_id): + # TODO(termie): validate that this request is valid for given user + # tenant + cred_ref = {'user_id': user_id, + 'tenant_id': tenant_id, + 'id': uuid.uuid4().hex, + 'secret': uuid.uuid4().hex} + self.ec2_api.create_credential(context, cred_ref['id'], cred_ref) + return cred_ref + + def get_credentials(self, context, user_id): + """List credentials for the given user_id.""" + # TODO(termie): validate that this request is valid for given user + # tenant + return self.ec2_api.list_credentials(user_id) + + def get_credential(self, context, user_id, credential_id): + # TODO(termie): validate that this request is valid for given user + # tenant + return self.ec2_api.get_credential(credential_id) + + def delete_credential(self, context, user_id, credential_id): + # TODO(termie): validate that this request is valid for given user + # tenant + return self.ec2_api.delete_credential(credential_id) + + class NoopController(Application): def __init__(self): super(NoopController, self).__init__() @@ -462,9 +588,6 @@ class TokenController(Application): logging.debug('TOKEN_REF %s', token_ref) return self._format_authenticate(token_ref, roles_ref, catalog_ref) - def authenticate_ec2(self, context): - raise NotImplemented() - # admin only def validate_token(self, context, token_id, belongs_to=None): """Check that a token is valid. diff --git a/keystone/utils.py b/keystone/utils.py index 9679dde249..869141abdb 100644 --- a/keystone/utils.py +++ b/keystone/utils.py @@ -17,9 +17,13 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 +import hashlib +import hmac import json import subprocess import sys +import urllib from keystone import logging @@ -53,6 +57,83 @@ class SmarterEncoder(json.JSONEncoder): return super(SmarterEncoder, self).default(obj) +class Ec2Signer(object): + """Hacked up code from boto/connection.py""" + + def __init__(self, secret_key): + secret_key = secret_key.encode() + self.hmac = hmac.new(secret_key, digestmod=hashlib.sha1) + if hashlib.sha256: + self.hmac_256 = hmac.new(secret_key, digestmod=hashlib.sha256) + + def generate(self, credentials): + """Generate auth string according to what SignatureVersion is given.""" + if credentials.params['SignatureVersion'] == '0': + return self._calc_signature_0(credentials.params) + if credentials.params['SignatureVersion'] == '1': + return self._calc_signature_1(credentials.params) + if credentials.params['SignatureVersion'] == '2': + return self._calc_signature_2(credentials.params, + credentials.verb, + credentials.host, + credentials.path) + raise Exception('Unknown Signature Version: %s' % + credentials.params['SignatureVersion']) + + @staticmethod + def _get_utf8_value(value): + """Get the UTF8-encoded version of a value.""" + if not isinstance(value, str) and not isinstance(value, unicode): + value = str(value) + if isinstance(value, unicode): + return value.encode('utf-8') + else: + return value + + def _calc_signature_0(self, params): + """Generate AWS signature version 0 string.""" + s = params['Action'] + params['Timestamp'] + self.hmac.update(s) + return base64.b64encode(self.hmac.digest()) + + def _calc_signature_1(self, params): + """Generate AWS signature version 1 string.""" + keys = params.keys() + keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) + for key in keys: + self.hmac.update(key) + val = self._get_utf8_value(params[key]) + self.hmac.update(val) + return base64.b64encode(self.hmac.digest()) + + def _calc_signature_2(self, params, verb, server_string, path): + """Generate AWS signature version 2 string.""" + LOG.debug('using _calc_signature_2') + string_to_sign = '%s\n%s\n%s\n' % (verb, server_string, path) + if self.hmac_256: + current_hmac = self.hmac_256 + params['SignatureMethod'] = 'HmacSHA256' + else: + current_hmac = self.hmac + params['SignatureMethod'] = 'HmacSHA1' + keys = params.keys() + keys.sort() + pairs = [] + for key in keys: + val = self._get_utf8_value(params[key]) + val = urllib.quote(val, safe='-_~') + pairs.append(urllib.quote(key, safe='') + '=' + val) + qs = '&'.join(pairs) + LOG.debug('query string: %s', qs) + string_to_sign += qs + LOG.debug('string_to_sign: %s', string_to_sign) + current_hmac.update(string_to_sign) + b64 = base64.b64encode(current_hmac.digest()) + LOG.debug('len(b64)=%d', len(b64)) + LOG.debug('base64 encoded digest: %s', b64) + return b64 + + # From python 2.7 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. From a0c7c7ce5104075bdf85d9e53e8382c5bb22cc92 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 16 Jan 2012 16:56:01 -0800 Subject: [PATCH 185/334] add sql backend, too --- keystone/backends/sql/core.py | 53 ++++++++++++++++++++++++++++++++++- keystone/service.py | 4 +-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index bfc3a97300..3c981b299b 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -151,6 +151,21 @@ class Token(Base, DictBase): data = sql.Column(JsonBlob()) +class Ec2Credential(Base, DictBase): + __tablename__ = 'ec2_credential' + access = sql.Column(sql.String(64), primary_key=True) + secret = sql.Column(sql.String(64)) + user_id = sql.Column(sql.String(64)) + tenant_id = sql.Column(sql.String(64)) + + @classmethod + def from_dict(cls, user_dict): + return cls(**user_dict) + + def to_dict(self): + return dict(self.iteritems()) + + class UserTenantMembership(Base, DictBase): """Tenant membership join table.""" __tablename__ = 'user_tenant_membership' @@ -206,7 +221,7 @@ class SqlIdentity(SqlBase): def db_sync(self): migration.db_sync() - # Identity interface + # Identity interface def authenticate(self, user_id=None, tenant_id=None, password=None): """Authenticate based on a user, tenant and password. @@ -464,3 +479,39 @@ class SqlToken(SqlBase): class SqlCatalog(SqlBase): pass + + +class SqlEc2(SqlBase): + # Internal interface to manage the database + def db_sync(self): + migration.db_sync() + + def get_credential(self, credential_id): + session = self.get_session() + credential_ref = session.query(Ec2Credential)\ + .filter_by(access=credential_id).first() + if not credential_ref: + return + return credential_ref.to_dict() + + def list_credentials(self): + session = self.get_session() + credential_refs = session.query(Ec2Credential) + return [x.to_dict() for x in credential_refs] + + # CRUD + def create_credential(self, credential_id, credential): + session = self.get_session() + with session.begin(): + credential_ref = Ec2Credential.from_dict(credential) + session.add(credential_ref) + session.flush() + return credential_ref.to_dict() + + def delete_credential(self, credential_id): + session = self.get_session() + credential_ref = session.query(Ec2Credential)\ + .filter_by(access=credential_id).first() + with session.begin(): + session.delete(credential_ref) + session.flush() diff --git a/keystone/service.py b/keystone/service.py index fe466d58e1..aed3b9297b 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -433,9 +433,9 @@ class Ec2Controller(Application): # tenant cred_ref = {'user_id': user_id, 'tenant_id': tenant_id, - 'id': uuid.uuid4().hex, + 'access': uuid.uuid4().hex, 'secret': uuid.uuid4().hex} - self.ec2_api.create_credential(context, cred_ref['id'], cred_ref) + self.ec2_api.create_credential(context, cred_ref['access'], cred_ref) return cred_ref def get_credentials(self, context, user_id): From dae746d9b779905c95efddcea066771bcddf1dad Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 16 Jan 2012 17:02:31 -0800 Subject: [PATCH 186/334] add keystoneclient expected format --- keystone/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/keystone/service.py b/keystone/service.py index aed3b9297b..138b86218b 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -436,18 +436,18 @@ class Ec2Controller(Application): 'access': uuid.uuid4().hex, 'secret': uuid.uuid4().hex} self.ec2_api.create_credential(context, cred_ref['access'], cred_ref) - return cred_ref + return {'credential': cred_ref} def get_credentials(self, context, user_id): """List credentials for the given user_id.""" # TODO(termie): validate that this request is valid for given user # tenant - return self.ec2_api.list_credentials(user_id) + return {'credentials': self.ec2_api.list_credentials(user_id)} def get_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user # tenant - return self.ec2_api.get_credential(credential_id) + return {'credential': self.ec2_api.get_credential(credential_id)} def delete_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user From 14189256690e2fd281a583383bcc23bbc489a8ab Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 16 Jan 2012 17:36:38 -0800 Subject: [PATCH 187/334] tests for ec2 crud --- etc/keystone.conf | 2 +- keystone/backends/kvs.py | 5 +++-- keystone/backends/sql/core.py | 5 +++-- keystone/service.py | 16 ++++++++-------- tests/test_keystoneclient.py | 21 +++++++++++++++++++++ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/etc/keystone.conf b/etc/keystone.conf index b937be45b5..f4c38e072c 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -66,7 +66,7 @@ paste.app_factory = keystone.service:admin_app_factory pipeline = token_auth admin_token_auth json_body debug ec2_extension public_service [pipeline:admin_api] -pipeline = token_auth admin_token_auth json_body debug crud_extension admin_service +pipeline = token_auth admin_token_auth json_body debug ec2_extension crud_extension admin_service [composite:main] use = egg:Paste#urlmap diff --git a/keystone/backends/kvs.py b/keystone/backends/kvs.py index 3ab5f1e9a4..f6771cb4f9 100644 --- a/keystone/backends/kvs.py +++ b/keystone/backends/kvs.py @@ -280,9 +280,10 @@ class KvsEc2(object): credential_ref = self.db.get('credential-%s' % credential_id) return credential_ref - def list_credentials(self): + def list_credentials(self, user_id): credential_ids = self.db.get('credential_list', []) - return [self.get_credential(x) for x in credential_ids] + rv = [self.get_credential(x) for x in credential_ids] + return [x for x in rv if x['user_id'] == user_id] # CRUD def create_credential(self, credential_id, credential): diff --git a/keystone/backends/sql/core.py b/keystone/backends/sql/core.py index 3c981b299b..0f870ca062 100644 --- a/keystone/backends/sql/core.py +++ b/keystone/backends/sql/core.py @@ -494,9 +494,10 @@ class SqlEc2(SqlBase): return return credential_ref.to_dict() - def list_credentials(self): + def list_credentials(self, user_id): session = self.get_session() - credential_refs = session.query(Ec2Credential) + credential_refs = session.query(Ec2Credential)\ + .filter_by(user_id=user_id) return [x.to_dict() for x in credential_refs] # CRUD diff --git a/keystone/service.py b/keystone/service.py index 138b86218b..1a618dba6f 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -335,25 +335,25 @@ class Ec2Extension(wsgi.ExtensionRouter): mapper.connect('/ec2tokens', controller=ec2_controller, action='authenticate_ec2', - conditions=dict(methods=['POST'])) + conditions=dict(method=['POST'])) # crud mapper.connect('/users/{user_id}/credentials/OS-EC2', controller=ec2_controller, action='create_credential', - conditions=dict(methods=['POST'])) + conditions=dict(method=['POST'])) mapper.connect('/users/{user_id}/credentials/OS-EC2', controller=ec2_controller, action='get_credentials', - conditions=dict(methods=['GET'])) + conditions=dict(method=['GET'])) mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', controller=ec2_controller, action='get_credential', - conditions=dict(methods=['GET'])) + conditions=dict(method=['GET'])) mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', controller=ec2_controller, action='delete_credential', - conditions=dict(methods=['DELETE'])) + conditions=dict(method=['DELETE'])) super(Ec2Extension, self).__init__(application, mapper) @@ -442,17 +442,17 @@ class Ec2Controller(Application): """List credentials for the given user_id.""" # TODO(termie): validate that this request is valid for given user # tenant - return {'credentials': self.ec2_api.list_credentials(user_id)} + return {'credentials': self.ec2_api.list_credentials(context, user_id)} def get_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user # tenant - return {'credential': self.ec2_api.get_credential(credential_id)} + return {'credential': self.ec2_api.get_credential(context, credential_id)} def delete_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user # tenant - return self.ec2_api.delete_credential(credential_id) + return self.ec2_api.delete_credential(context, credential_id) class NoopController(Application): diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index bb568b0963..3fd5edca15 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -239,6 +239,27 @@ class KcMasterTestCase(CompatTestCase): roles = client.roles.get_user_role_refs('foo') self.assertTrue(len(roles) > 0) + def test_ec2_credential_creation(self): + from keystoneclient import exceptions as client_exceptions + + client = self.foo_client() + creds = client.ec2.list(self.user_foo['id']) + self.assertEquals(creds, []) + + cred = client.ec2.create(self.user_foo['id'], self.tenant_bar['id']) + creds = client.ec2.list(self.user_foo['id']) + self.assertEquals(creds, [cred]) + + got = client.ec2.get(self.user_foo['id'], cred.access) + self.assertEquals(cred, got) + + # FIXME(ja): need to test ec2 validation here + + client.ec2.delete(self.user_foo['id'], cred.access) + creds = client.ec2.list(self.user_foo['id']) + self.assertEquals(creds, []) + + def test_service_create_and_delete(self): from keystoneclient import exceptions as client_exceptions From 44c6b6939b0b6a734631c9859f7fdce40dedb31e Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 16 Jan 2012 17:38:02 -0800 Subject: [PATCH 188/334] spacing --- tests/test_keystoneclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 3fd5edca15..3eeed6f61b 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -259,7 +259,6 @@ class KcMasterTestCase(CompatTestCase): creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) - def test_service_create_and_delete(self): from keystoneclient import exceptions as client_exceptions From f16a262c916fe03a7301a9968fdd99737644bbbb Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 16 Jan 2012 22:52:52 -0800 Subject: [PATCH 189/334] return to starting directory after git work --- keystone/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/keystone/test.py b/keystone/test.py index 7ef1b6ede0..b3473b5486 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -44,6 +44,7 @@ def checkout_vendor(repo, rev): if name.endswith('.git'): name = name[:-4] + working_dir = os.getcwd() revdir = os.path.join(VENDOR, '%s-%s' % (name, rev.replace('/', '_'))) modcheck = os.path.join(VENDOR, '.%s-%s' % (name, rev.replace('/', '_'))) try: @@ -64,6 +65,7 @@ def checkout_vendor(repo, rev): fd.write('1') except subprocess.CalledProcessError as e: logging.warning('Failed to checkout %s', repo) + cd(working_dir) return revdir From 53ec23ab7384ddf7e2ad37f1e7260ebbb4956402 Mon Sep 17 00:00:00 2001 From: vishvananda Date: Tue, 17 Jan 2012 13:25:37 -0800 Subject: [PATCH 190/334] Fix typo --- keystone/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/service.py b/keystone/service.py index 1a618dba6f..8119a80db1 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -371,7 +371,7 @@ class Ec2Controller(Application): ec2Credentials=None): """Validate a signed EC2 request and provide a token.""" # NOTE(termie): backwards compat hack - if not ecredentials and ec2Credentials: + if not credentials and ec2Credentials: credentials = ec2Credentials creds_ref = self.ec2_api.get_credential(context, credentials['access']) From 9d04ee987bff0bcf7587b632fdea86518c4adc9a Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 17 Jan 2012 13:29:10 -0800 Subject: [PATCH 191/334] rename apidoc to autodoc --- docs/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 5083ea0d1b..e0366c8588 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,6 +20,7 @@ I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source help: @echo "Please use \`make ' where is one of" + @echo " autodoc generate the autodoc templates" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @@ -43,7 +44,7 @@ help: clean: -rm -rf $(BUILDDIR)/* -apidoc: +autodoc: $(SPHINXAPIDOC) -o $(SOURCEDIR) ../keystone html: From 3c10e730839bb313ad33228eb56afcd3a8e55bc1 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 17 Jan 2012 13:29:18 -0800 Subject: [PATCH 192/334] fix pep8 error --- keystone/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keystone/service.py b/keystone/service.py index 1a618dba6f..3cf5f8a79e 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -447,7 +447,8 @@ class Ec2Controller(Application): def get_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user # tenant - return {'credential': self.ec2_api.get_credential(context, credential_id)} + return {'credential': self.ec2_api.get_credential(context, + credential_id)} def delete_credential(self, context, user_id, credential_id): # TODO(termie): validate that this request is valid for given user From 76c45b4791275daccff7545f677684fa9a695431 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 17 Jan 2012 16:05:31 -0800 Subject: [PATCH 193/334] make a main in keystone-manage --- bin/keystone-manage | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index 310b5bfadd..63c1aac12f 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -131,7 +131,10 @@ CMDS = {'db_sync': DbSync, } -if __name__ == '__main__': +def main(argv=None): + if argv is None: + argv = sys.argv + dev_conf = os.path.join(possible_topdir, 'etc', 'keystone.conf') @@ -139,7 +142,11 @@ if __name__ == '__main__': if os.path.exists(dev_conf): config_files = [dev_conf] - args = CONF(config_files=config_files, args=sys.argv) + args = CONF(config_files=config_files, args=argv) cmd = args[1] if cmd in CMDS: CMDS[cmd](argv=(args[:1] + args[2:])).run() + + +if __name__ == '__main__': + main() From e75f7bebdff2df2342178270efe39ebf38d0672f Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 17 Jan 2012 17:36:03 -0800 Subject: [PATCH 194/334] needed to do more for cli opts --- keystone/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/config.py b/keystone/config.py index bc5a2683ce..f3064dc7be 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -82,7 +82,7 @@ def register_str(*args, **kw): def register_cli_str(*args, **kw): group = _ensure_group(kw) - return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) def register_bool(*args, **kw): From 67d4a7c624c166269e0b2f1aeb65b32ceb5ab645 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Wed, 18 Jan 2012 01:33:44 +0000 Subject: [PATCH 195/334] updating dependencies for ksl --- tools/pip-requires | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pip-requires b/tools/pip-requires index 3352ed6645..c9215f11fb 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -7,3 +7,6 @@ routes pycli sqlalchemy sqlalchemy-migrate +pycli + +-e git+https://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient From 198d168a529b5aedc4b3b814748f43980584ff49 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 17 Jan 2012 13:38:41 -0800 Subject: [PATCH 196/334] testing rst on github --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 46c3ec6004..d01098731d 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,8 @@ -Keystone -======== +.. image:: http://term.ie/data/medium_ksl.png + :alt: Keystone + +.. toctree:: + :maxdepth 2 Keystone is an OpenStack project that provides Identity, Token, Catalog and Policy services for use specifically by projects in the OpenStack family. From e129d5f402255bd10e2bd2f8a80284b7b38ab749 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 10:54:33 -0800 Subject: [PATCH 197/334] fix sphinx --- docs/Makefile | 2 +- docs/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index e0366c8588..03e3ef3bc3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -45,7 +45,7 @@ clean: -rm -rf $(BUILDDIR)/* autodoc: - $(SPHINXAPIDOC) -o $(SOURCEDIR) ../keystone + $(SPHINXAPIDOC) -f -o $(SOURCEDIR) ../keystone html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/source/conf.py b/docs/source/conf.py index 1f7785ef0a..9c18c5ff69 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('../..')) # -- General configuration ----------------------------------------------------- From bf7e6fb47d8587e4c5176c5281fb3bbe724a85a4 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 10:54:48 -0800 Subject: [PATCH 198/334] some tiny docs --- keystone/identity.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/keystone/identity.py b/keystone/identity.py index 6ef8298a6a..449f01d9c8 100644 --- a/keystone/identity.py +++ b/keystone/identity.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +"""Main entry point into the Identity service.""" + from keystone import config from keystone import manager @@ -8,5 +10,9 @@ CONF = config.CONF class Manager(manager.Manager): + """Default pivot point for the Identity backend. + + See :mod:`keystone.manager.Manager` for more details. + """ def __init__(self): super(Manager, self).__init__(CONF.identity.driver) From 94f78a3082efb7a73a8670701e9c037b7ffde581 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 11:36:02 -0800 Subject: [PATCH 199/334] expect sphinx sources to be autogenned --- .gitignore | 2 + docs/source/keystone.backends.rst | 42 ------ .../keystone.backends.sql.migrate_repo.rst | 18 --- ...one.backends.sql.migrate_repo.versions.rst | 11 -- docs/source/keystone.backends.sql.rst | 42 ------ docs/source/keystone.rst | 130 ------------------ docs/source/modules.rst | 9 -- 7 files changed, 2 insertions(+), 252 deletions(-) delete mode 100644 docs/source/keystone.backends.rst delete mode 100644 docs/source/keystone.backends.sql.migrate_repo.rst delete mode 100644 docs/source/keystone.backends.sql.migrate_repo.versions.rst delete mode 100644 docs/source/keystone.backends.sql.rst delete mode 100644 docs/source/keystone.rst delete mode 100644 docs/source/modules.rst diff --git a/.gitignore b/.gitignore index 3d40983549..6dc19a6eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ nosetests.xml bla.db docs/build .DS_Store +docs/source/modules.rst +docs/source/keystone.* diff --git a/docs/source/keystone.backends.rst b/docs/source/keystone.backends.rst deleted file mode 100644 index 8641ad9e16..0000000000 --- a/docs/source/keystone.backends.rst +++ /dev/null @@ -1,42 +0,0 @@ -backends Package -================ - -:mod:`kvs` Module ------------------ - -.. automodule:: keystone.backends.kvs - :members: - :undoc-members: - :show-inheritance: - -:mod:`pam` Module ------------------ - -.. automodule:: keystone.backends.pam - :members: - :undoc-members: - :show-inheritance: - -:mod:`policy` Module --------------------- - -.. automodule:: keystone.backends.policy - :members: - :undoc-members: - :show-inheritance: - -:mod:`templated` Module ------------------------ - -.. automodule:: keystone.backends.templated - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - keystone.backends.sql - diff --git a/docs/source/keystone.backends.sql.migrate_repo.rst b/docs/source/keystone.backends.sql.migrate_repo.rst deleted file mode 100644 index d47a101eaa..0000000000 --- a/docs/source/keystone.backends.sql.migrate_repo.rst +++ /dev/null @@ -1,18 +0,0 @@ -migrate_repo Package -==================== - -:mod:`manage` Module --------------------- - -.. automodule:: keystone.backends.sql.migrate_repo.manage - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - keystone.backends.sql.migrate_repo.versions - diff --git a/docs/source/keystone.backends.sql.migrate_repo.versions.rst b/docs/source/keystone.backends.sql.migrate_repo.versions.rst deleted file mode 100644 index 85e1ce8629..0000000000 --- a/docs/source/keystone.backends.sql.migrate_repo.versions.rst +++ /dev/null @@ -1,11 +0,0 @@ -versions Package -================ - -:mod:`001_add_initial_tables` Module ------------------------------------- - -.. automodule:: keystone.backends.sql.migrate_repo.versions.001_add_initial_tables - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/source/keystone.backends.sql.rst b/docs/source/keystone.backends.sql.rst deleted file mode 100644 index 91c9f735dd..0000000000 --- a/docs/source/keystone.backends.sql.rst +++ /dev/null @@ -1,42 +0,0 @@ -sql Package -=========== - -:mod:`sql` Package ------------------- - -.. automodule:: keystone.backends.sql - :members: - :undoc-members: - :show-inheritance: - -:mod:`core` Module ------------------- - -.. automodule:: keystone.backends.sql.core - :members: - :undoc-members: - :show-inheritance: - -:mod:`migration` Module ------------------------ - -.. automodule:: keystone.backends.sql.migration - :members: - :undoc-members: - :show-inheritance: - -:mod:`util` Module ------------------- - -.. automodule:: keystone.backends.sql.util - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - keystone.backends.sql.migrate_repo - diff --git a/docs/source/keystone.rst b/docs/source/keystone.rst deleted file mode 100644 index d4737f0672..0000000000 --- a/docs/source/keystone.rst +++ /dev/null @@ -1,130 +0,0 @@ -keystone Package -================ - -:mod:`catalog` Module ---------------------- - -.. automodule:: keystone.catalog - :members: - :undoc-members: - :show-inheritance: - -:mod:`cfg` Module ------------------ - -.. automodule:: keystone.cfg - :members: - :undoc-members: - :show-inheritance: - -:mod:`client` Module --------------------- - -.. automodule:: keystone.client - :members: - :undoc-members: - :show-inheritance: - -:mod:`config` Module --------------------- - -.. automodule:: keystone.config - :members: - :undoc-members: - :show-inheritance: - -:mod:`identity` Module ----------------------- - -.. automodule:: keystone.identity - :members: - :undoc-members: - :show-inheritance: - -:mod:`keystone_compat` Module ------------------------------ - -.. automodule:: keystone.keystone_compat - :members: - :undoc-members: - :show-inheritance: - -:mod:`logging` Module ---------------------- - -.. automodule:: keystone.logging - :members: - :undoc-members: - :show-inheritance: - -:mod:`middleware` Module ------------------------- - -.. automodule:: keystone.middleware - :members: - :undoc-members: - :show-inheritance: - -:mod:`models` Module --------------------- - -.. automodule:: keystone.models - :members: - :undoc-members: - :show-inheritance: - -:mod:`policy` Module --------------------- - -.. automodule:: keystone.policy - :members: - :undoc-members: - :show-inheritance: - -:mod:`service` Module ---------------------- - -.. automodule:: keystone.service - :members: - :undoc-members: - :show-inheritance: - -:mod:`test` Module ------------------- - -.. automodule:: keystone.test - :members: - :undoc-members: - :show-inheritance: - -:mod:`token` Module -------------------- - -.. automodule:: keystone.token - :members: - :undoc-members: - :show-inheritance: - -:mod:`utils` Module -------------------- - -.. automodule:: keystone.utils - :members: - :undoc-members: - :show-inheritance: - -:mod:`wsgi` Module ------------------- - -.. automodule:: keystone.wsgi - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - keystone.backends - diff --git a/docs/source/modules.rst b/docs/source/modules.rst deleted file mode 100644 index e071e10801..0000000000 --- a/docs/source/modules.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. -== - -.. toctree:: - :maxdepth: 4 - - keystone - run_tests - setup From f0e3e7f123c24f4441de14be34e5b9fd307354aa Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 11:40:36 -0800 Subject: [PATCH 200/334] add docs for various service managers --- keystone/catalog.py | 12 ++++++++++++ keystone/ec2.py | 12 ++++++++++++ keystone/identity.py | 8 +++++++- keystone/policy.py | 12 ++++++++++++ keystone/token.py | 12 ++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/keystone/catalog.py b/keystone/catalog.py index 8382108d1f..a76ba6edb4 100644 --- a/keystone/catalog.py +++ b/keystone/catalog.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +"""Main entry point into the Catalog service.""" + from keystone import config from keystone import manager @@ -8,5 +10,15 @@ CONF = config.CONF class Manager(manager.Manager): + """Default pivot point for the Catalog backend. + + See :mod:`keystone.manager.Manager` for more details on how this + dynamically calls the backend. + + See :mod:`keystone.backends.base.Catalog` for more details on the + interface provided by backends. + + """ + def __init__(self): super(Manager, self).__init__(CONF.catalog.driver) diff --git a/keystone/ec2.py b/keystone/ec2.py index a0f1c0b30e..a02e4daf22 100644 --- a/keystone/ec2.py +++ b/keystone/ec2.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +"""Main entry point into the EC2 Credentials service.""" + from keystone import config from keystone import manager @@ -8,5 +10,15 @@ CONF = config.CONF class Manager(manager.Manager): + """Default pivot point for the EC2 Credentials backend. + + See :mod:`keystone.manager.Manager` for more details on how this + dynamically calls the backend. + + See :mod:`keystone.backends.base.Ec2` for more details on the + interface provided by backends. + + """ + def __init__(self): super(Manager, self).__init__(CONF.ec2.driver) diff --git a/keystone/identity.py b/keystone/identity.py index 449f01d9c8..5f25fbf47b 100644 --- a/keystone/identity.py +++ b/keystone/identity.py @@ -12,7 +12,13 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the Identity backend. - See :mod:`keystone.manager.Manager` for more details. + See :mod:`keystone.manager.Manager` for more details on how this + dynamically calls the backend. + + See :mod:`keystone.backends.base.Identity` for more details on the + interface provided by backends. + """ + def __init__(self): super(Manager, self).__init__(CONF.identity.driver) diff --git a/keystone/policy.py b/keystone/policy.py index f41f54ad78..4aa24df759 100644 --- a/keystone/policy.py +++ b/keystone/policy.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +"""Main entry point into the Policy service.""" + from keystone import config from keystone import manager @@ -8,5 +10,15 @@ CONF = config.CONF class Manager(manager.Manager): + """Default pivot point for the Policy backend. + + See :mod:`keystone.manager.Manager` for more details on how this + dynamically calls the backend. + + See :mod:`keystone.backends.base.Policy` for more details on the + interface provided by backends. + + """ + def __init__(self): super(Manager, self).__init__(CONF.policy.driver) diff --git a/keystone/token.py b/keystone/token.py index 0657f8fac9..d3e9a5f09a 100644 --- a/keystone/token.py +++ b/keystone/token.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +"""Main entry point into the Token service.""" + from keystone import config from keystone import manager @@ -8,5 +10,15 @@ CONF = config.CONF class Manager(manager.Manager): + """Default pivot point for the Token backend. + + See :mod:`keystone.manager.Manager` for more details on how this + dynamically calls the backend. + + See :mod:`keystone.backends.base.Token` for more details on the + interface provided by backends. + + """ + def __init__(self): super(Manager, self).__init__(CONF.token.driver) From 909012a63e5761b4ff09d2c63a7a0493f7daffa6 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 20:06:27 -0800 Subject: [PATCH 201/334] establish basic structure --- keystone/{backends => catalog}/__init__.py | 0 .../{backends/sql/migrate_repo => catalog/backends}/__init__.py | 0 keystone/{ => catalog}/backends/templated.py | 0 keystone/{catalog.py => catalog/core.py} | 0 .../{backends/sql/migrate_repo/versions => common}/__init__.py | 0 keystone/{ => common}/bufferedhttp.py | 0 keystone/{ => common}/cfg.py | 0 keystone/{backends => common}/kvs.py | 0 keystone/{ => common}/logging.py | 0 keystone/{ => common}/manager.py | 0 keystone/{backends => common}/sql/__init__.py | 0 keystone/{backends => common}/sql/core.py | 0 keystone/{backends => common}/sql/migrate_repo/README | 0 keystone/common/sql/migrate_repo/__init__.py | 0 keystone/{backends => common}/sql/migrate_repo/manage.py | 0 keystone/{backends => common}/sql/migrate_repo/migrate.cfg | 0 .../sql/migrate_repo/versions/001_add_initial_tables.py | 0 keystone/common/sql/migrate_repo/versions/__init__.py | 0 keystone/{backends => common}/sql/migration.py | 0 keystone/{backends => common}/sql/util.py | 0 keystone/{ => common}/utils.py | 0 keystone/{ => common}/wsgi.py | 0 keystone/contrib/__init__.py | 0 keystone/contrib/admin_crud/__init__.py | 0 keystone/contrib/ec2/__init__.py | 0 keystone/{ec2.py => contrib/ec2/core.py} | 0 keystone/identity/__init__.py | 0 keystone/identity/backends/__init__.py | 0 keystone/{ => identity}/backends/pam.py | 0 keystone/{identity.py => identity/core.py} | 0 keystone/policy/__init__.py | 0 keystone/policy/backends/__init__.py | 0 keystone/{backends/policy.py => policy/backends/simple.py} | 0 keystone/{policy.py => policy/core.py} | 0 keystone/token/__init__.py | 0 keystone/{token.py => token/core.py} | 0 36 files changed, 0 insertions(+), 0 deletions(-) rename keystone/{backends => catalog}/__init__.py (100%) rename keystone/{backends/sql/migrate_repo => catalog/backends}/__init__.py (100%) rename keystone/{ => catalog}/backends/templated.py (100%) rename keystone/{catalog.py => catalog/core.py} (100%) rename keystone/{backends/sql/migrate_repo/versions => common}/__init__.py (100%) rename keystone/{ => common}/bufferedhttp.py (100%) rename keystone/{ => common}/cfg.py (100%) rename keystone/{backends => common}/kvs.py (100%) rename keystone/{ => common}/logging.py (100%) rename keystone/{ => common}/manager.py (100%) rename keystone/{backends => common}/sql/__init__.py (100%) rename keystone/{backends => common}/sql/core.py (100%) rename keystone/{backends => common}/sql/migrate_repo/README (100%) create mode 100644 keystone/common/sql/migrate_repo/__init__.py rename keystone/{backends => common}/sql/migrate_repo/manage.py (100%) rename keystone/{backends => common}/sql/migrate_repo/migrate.cfg (100%) rename keystone/{backends => common}/sql/migrate_repo/versions/001_add_initial_tables.py (100%) create mode 100644 keystone/common/sql/migrate_repo/versions/__init__.py rename keystone/{backends => common}/sql/migration.py (100%) rename keystone/{backends => common}/sql/util.py (100%) rename keystone/{ => common}/utils.py (100%) rename keystone/{ => common}/wsgi.py (100%) create mode 100644 keystone/contrib/__init__.py create mode 100644 keystone/contrib/admin_crud/__init__.py create mode 100644 keystone/contrib/ec2/__init__.py rename keystone/{ec2.py => contrib/ec2/core.py} (100%) create mode 100644 keystone/identity/__init__.py create mode 100644 keystone/identity/backends/__init__.py rename keystone/{ => identity}/backends/pam.py (100%) rename keystone/{identity.py => identity/core.py} (100%) create mode 100644 keystone/policy/__init__.py create mode 100644 keystone/policy/backends/__init__.py rename keystone/{backends/policy.py => policy/backends/simple.py} (100%) rename keystone/{policy.py => policy/core.py} (100%) create mode 100644 keystone/token/__init__.py rename keystone/{token.py => token/core.py} (100%) diff --git a/keystone/backends/__init__.py b/keystone/catalog/__init__.py similarity index 100% rename from keystone/backends/__init__.py rename to keystone/catalog/__init__.py diff --git a/keystone/backends/sql/migrate_repo/__init__.py b/keystone/catalog/backends/__init__.py similarity index 100% rename from keystone/backends/sql/migrate_repo/__init__.py rename to keystone/catalog/backends/__init__.py diff --git a/keystone/backends/templated.py b/keystone/catalog/backends/templated.py similarity index 100% rename from keystone/backends/templated.py rename to keystone/catalog/backends/templated.py diff --git a/keystone/catalog.py b/keystone/catalog/core.py similarity index 100% rename from keystone/catalog.py rename to keystone/catalog/core.py diff --git a/keystone/backends/sql/migrate_repo/versions/__init__.py b/keystone/common/__init__.py similarity index 100% rename from keystone/backends/sql/migrate_repo/versions/__init__.py rename to keystone/common/__init__.py diff --git a/keystone/bufferedhttp.py b/keystone/common/bufferedhttp.py similarity index 100% rename from keystone/bufferedhttp.py rename to keystone/common/bufferedhttp.py diff --git a/keystone/cfg.py b/keystone/common/cfg.py similarity index 100% rename from keystone/cfg.py rename to keystone/common/cfg.py diff --git a/keystone/backends/kvs.py b/keystone/common/kvs.py similarity index 100% rename from keystone/backends/kvs.py rename to keystone/common/kvs.py diff --git a/keystone/logging.py b/keystone/common/logging.py similarity index 100% rename from keystone/logging.py rename to keystone/common/logging.py diff --git a/keystone/manager.py b/keystone/common/manager.py similarity index 100% rename from keystone/manager.py rename to keystone/common/manager.py diff --git a/keystone/backends/sql/__init__.py b/keystone/common/sql/__init__.py similarity index 100% rename from keystone/backends/sql/__init__.py rename to keystone/common/sql/__init__.py diff --git a/keystone/backends/sql/core.py b/keystone/common/sql/core.py similarity index 100% rename from keystone/backends/sql/core.py rename to keystone/common/sql/core.py diff --git a/keystone/backends/sql/migrate_repo/README b/keystone/common/sql/migrate_repo/README similarity index 100% rename from keystone/backends/sql/migrate_repo/README rename to keystone/common/sql/migrate_repo/README diff --git a/keystone/common/sql/migrate_repo/__init__.py b/keystone/common/sql/migrate_repo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/backends/sql/migrate_repo/manage.py b/keystone/common/sql/migrate_repo/manage.py similarity index 100% rename from keystone/backends/sql/migrate_repo/manage.py rename to keystone/common/sql/migrate_repo/manage.py diff --git a/keystone/backends/sql/migrate_repo/migrate.cfg b/keystone/common/sql/migrate_repo/migrate.cfg similarity index 100% rename from keystone/backends/sql/migrate_repo/migrate.cfg rename to keystone/common/sql/migrate_repo/migrate.cfg diff --git a/keystone/backends/sql/migrate_repo/versions/001_add_initial_tables.py b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py similarity index 100% rename from keystone/backends/sql/migrate_repo/versions/001_add_initial_tables.py rename to keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py diff --git a/keystone/common/sql/migrate_repo/versions/__init__.py b/keystone/common/sql/migrate_repo/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/backends/sql/migration.py b/keystone/common/sql/migration.py similarity index 100% rename from keystone/backends/sql/migration.py rename to keystone/common/sql/migration.py diff --git a/keystone/backends/sql/util.py b/keystone/common/sql/util.py similarity index 100% rename from keystone/backends/sql/util.py rename to keystone/common/sql/util.py diff --git a/keystone/utils.py b/keystone/common/utils.py similarity index 100% rename from keystone/utils.py rename to keystone/common/utils.py diff --git a/keystone/wsgi.py b/keystone/common/wsgi.py similarity index 100% rename from keystone/wsgi.py rename to keystone/common/wsgi.py diff --git a/keystone/contrib/__init__.py b/keystone/contrib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/contrib/admin_crud/__init__.py b/keystone/contrib/admin_crud/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/contrib/ec2/__init__.py b/keystone/contrib/ec2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/ec2.py b/keystone/contrib/ec2/core.py similarity index 100% rename from keystone/ec2.py rename to keystone/contrib/ec2/core.py diff --git a/keystone/identity/__init__.py b/keystone/identity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/identity/backends/__init__.py b/keystone/identity/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/backends/pam.py b/keystone/identity/backends/pam.py similarity index 100% rename from keystone/backends/pam.py rename to keystone/identity/backends/pam.py diff --git a/keystone/identity.py b/keystone/identity/core.py similarity index 100% rename from keystone/identity.py rename to keystone/identity/core.py diff --git a/keystone/policy/__init__.py b/keystone/policy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/policy/backends/__init__.py b/keystone/policy/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/backends/policy.py b/keystone/policy/backends/simple.py similarity index 100% rename from keystone/backends/policy.py rename to keystone/policy/backends/simple.py diff --git a/keystone/policy.py b/keystone/policy/core.py similarity index 100% rename from keystone/policy.py rename to keystone/policy/core.py diff --git a/keystone/token/__init__.py b/keystone/token/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/token.py b/keystone/token/core.py similarity index 100% rename from keystone/token.py rename to keystone/token/core.py From 308a766b5b8ddbfbaa549e3af7fa736c1e1d4218 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 20:57:44 -0800 Subject: [PATCH 202/334] split up the services and kvs backends --- keystone/catalog/__init__.py | 1 + keystone/catalog/backends/kvs.py | 42 ++ keystone/catalog/backends/templated.py | 6 +- keystone/catalog/core.py | 41 +- keystone/common/kvs.py | 283 +-------- keystone/common/manager.py | 2 +- keystone/common/utils.py | 2 +- keystone/common/wsgi.py | 86 ++- keystone/contrib/admin_crud/__init__.py | 1 + keystone/contrib/admin_crud/core.py | 150 +++++ keystone/contrib/ec2/__init__.py | 1 + keystone/contrib/ec2/backends/__init__.py | 0 keystone/contrib/ec2/backends/kvs.py | 32 + keystone/contrib/ec2/core.py | 135 ++++- keystone/identity/__init__.py | 1 + keystone/identity/backends/kvs.py | 178 ++++++ keystone/identity/core.py | 293 ++++++++++ keystone/policy/backends/simple.py | 2 +- keystone/service.py | 681 +--------------------- keystone/token/__init__.py | 1 + keystone/token/backends/__init__.py | 0 keystone/token/backends/kvs.py | 15 + 22 files changed, 996 insertions(+), 957 deletions(-) create mode 100644 keystone/catalog/backends/kvs.py create mode 100644 keystone/contrib/admin_crud/core.py create mode 100644 keystone/contrib/ec2/backends/__init__.py create mode 100644 keystone/contrib/ec2/backends/kvs.py create mode 100644 keystone/identity/backends/kvs.py create mode 100644 keystone/token/backends/__init__.py create mode 100644 keystone/token/backends/kvs.py diff --git a/keystone/catalog/__init__.py b/keystone/catalog/__init__.py index e69de29bb2..95af1256fd 100644 --- a/keystone/catalog/__init__.py +++ b/keystone/catalog/__init__.py @@ -0,0 +1 @@ +from keystone.catalog.core import * diff --git a/keystone/catalog/backends/kvs.py b/keystone/catalog/backends/kvs.py new file mode 100644 index 0000000000..84d1857254 --- /dev/null +++ b/keystone/catalog/backends/kvs.py @@ -0,0 +1,42 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + + +from keystone.common import kvs + + +class KvsCatalog(kvs.Base): + # Public interface + def get_catalog(self, user_id, tenant_id, metadata=None): + return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) + + def get_service(self, service_id): + return self.db.get('service-%s' % service_id) + + def list_services(self): + return self.db.get('service_list', []) + + def create_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) + service_list = set(self.db.get('service_list', [])) + service_list.add(service_id) + self.db.set('service_list', list(service_list)) + return service + + def update_service(self, service_id, service): + self.db.set('service-%s' % service_id, service) + return service + + def delete_service(self, service_id): + self.db.delete('service-%s' % service_id) + service_list = set(self.db.get('service_list', [])) + service_list.remove(service_id) + self.db.set('service_list', list(service_list)) + return None + + # Private interface + def _create_catalog(self, user_id, tenant_id, data): + self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) + return data + + + diff --git a/keystone/catalog/backends/templated.py b/keystone/catalog/backends/templated.py index 475f13daf7..92c52574a2 100644 --- a/keystone/catalog/backends/templated.py +++ b/keystone/catalog/backends/templated.py @@ -1,15 +1,15 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 from keystone import config -from keystone import logging -from keystone.backends import kvs +from keystone.common import logging +from keystone.catalog.backends import kvs CONF = config.CONF config.register_str('template_file', group='catalog') -class TemplatedCatalog(kvs.KvsCatalog): +class TemplatedCatalog(kvs.Catalog): """A backend that generates endpoints for the Catalog based on templates. It is usually configured via config entries that look like: diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index a76ba6edb4..8177f2cffe 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -3,7 +3,11 @@ """Main entry point into the Catalog service.""" from keystone import config -from keystone import manager +from keystone import identity +from keystone import token +from keystone import policy +from keystone.common import manager +from keystone.common import wsgi CONF = config.CONF @@ -22,3 +26,38 @@ class Manager(manager.Manager): def __init__(self): super(Manager, self).__init__(CONF.catalog.driver) + + +class ServiceController(wsgi.Application): + def __init__(self): + self.catalog_api = Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() + super(ServiceController, self).__init__() + + # CRUD extensions + # NOTE(termie): this OS-KSADM stuff is not very consistent + def get_services(self, context): + service_list = self.catalog_api.list_services(context) + service_refs = [self.catalog_api.get_service(context, x) + for x in service_list] + return {'OS-KSADM:services': service_refs} + + def get_service(self, context, service_id): + service_ref = self.catalog_api.get_service(context, service_id) + if not service_ref: + raise webob.exc.HTTPNotFound() + return {'OS-KSADM:service': service_ref} + + def delete_service(self, context, service_id): + service_ref = self.catalog_api.delete_service(context, service_id) + + def create_service(self, context, OS_KSADM_service): + service_id = uuid.uuid4().hex + service_ref = OS_KSADM_service.copy() + service_ref['id'] = service_id + new_service_ref = self.catalog_api.create_service( + context, service_id, service_ref) + return {'OS-KSADM:service': new_service_ref} + diff --git a/keystone/common/kvs.py b/keystone/common/kvs.py index f6771cb4f9..7f24d9ba61 100644 --- a/keystone/common/kvs.py +++ b/keystone/common/kvs.py @@ -12,291 +12,10 @@ class DictKvs(dict): INMEMDB = DictKvs() -class KvsIdentity(object): +class Base(object): def __init__(self, db=None): if db is None: db = INMEMDB elif type(db) is type({}): db = DictKvs(db) self.db = db - - # Public interface - def authenticate(self, user_id=None, tenant_id=None, password=None): - """Authenticate based on a user, tenant and password. - - Expects the user object to have a password field and the tenant to be - in the list of tenants on the user. - - """ - user_ref = self.get_user(user_id) - tenant_ref = None - metadata_ref = None - if not user_ref or user_ref.get('password') != password: - raise AssertionError('Invalid user / password') - if tenant_id and tenant_id not in user_ref['tenants']: - raise AssertionError('Invalid tenant') - - tenant_ref = self.get_tenant(tenant_id) - if tenant_ref: - metadata_ref = self.get_metadata(user_id, tenant_id) - else: - metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) - - def get_tenant(self, tenant_id): - tenant_ref = self.db.get('tenant-%s' % tenant_id) - return tenant_ref - - def get_tenant_by_name(self, tenant_name): - tenant_ref = self.db.get('tenant_name-%s' % tenant_name) - return tenant_ref - - def get_user(self, user_id): - user_ref = self.db.get('user-%s' % user_id) - return user_ref - - def get_user_by_name(self, user_name): - user_ref = self.db.get('user_name-%s' % user_name) - return user_ref - - def get_metadata(self, user_id, tenant_id): - return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) - - def get_role(self, role_id): - role_ref = self.db.get('role-%s' % role_id) - return role_ref - - def list_users(self): - user_ids = self.db.get('user_list', []) - return [self.get_user(x) for x in user_ids] - - def list_roles(self): - role_ids = self.db.get('role_list', []) - return [self.get_role(x) for x in role_ids] - - # These should probably be part of the high-level API - def add_user_to_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.add(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) - - def remove_user_from_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.remove(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) - - def get_tenants_for_user(self, user_id): - user_ref = self.get_user(user_id) - return user_ref.get('tenants', []) - - def get_roles_for_user_and_tenant(self, user_id, tenant_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - return metadata_ref.get('roles', []) - - def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.add(role_id) - metadata_ref['roles'] = list(roles) - self.update_metadata(user_id, tenant_id, metadata_ref) - - def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.remove(role_id) - metadata_ref['roles'] = list(roles) - self.update_metadata(user_id, tenant_id, metadata_ref) - - # CRUD - def create_user(self, user_id, user): - self.db.set('user-%s' % user_id, user) - self.db.set('user_name-%s' % user['name'], user) - user_list = set(self.db.get('user_list', [])) - user_list.add(user_id) - self.db.set('user_list', list(user_list)) - return user - - def update_user(self, user_id, user): - # get the old name and delete it too - old_user = self.db.get('user-%s' % user_id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % user_id, user) - self.db.set('user_name-%s' % user['name'], user) - return user - - def delete_user(self, user_id): - old_user = self.db.get('user-%s' % user_id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.delete('user-%s' % user_id) - user_list = set(self.db.get('user_list', [])) - user_list.remove(user_id) - self.db.set('user_list', list(user_list)) - return None - - def create_tenant(self, tenant_id, tenant): - self.db.set('tenant-%s' % tenant_id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant - - def update_tenant(self, tenant_id, tenant): - # get the old name and delete it too - old_tenant = self.db.get('tenant-%s' % tenant_id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % tenant_id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant - - def delete_tenant(self, tenant_id): - old_tenant = self.db.get('tenant-%s' % tenant_id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.delete('tenant-%s' % tenant_id) - return None - - def create_metadata(self, user_id, tenant_id, metadata): - self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) - return metadata - - def update_metadata(self, user_id, tenant_id, metadata): - self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) - return metadata - - def delete_metadata(self, user_id, tenant_id): - self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) - return None - - def create_role(self, role_id, role): - self.db.set('role-%s' % role_id, role) - role_list = set(self.db.get('role_list', [])) - role_list.add(role_id) - self.db.set('role_list', list(role_list)) - return role - - def update_role(self, role_id, role): - self.db.set('role-%s' % role_id, role) - return role - - def delete_role(self, role_id): - self.db.delete('role-%s' % role_id) - role_list = set(self.db.get('role_list', [])) - role_list.remove(role_id) - self.db.set('role_list', list(role_list)) - return None - - -class KvsToken(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db - - # Public interface - def get_token(self, token_id): - return self.db.get('token-%s' % token_id) - - def create_token(self, token_id, data): - self.db.set('token-%s' % token_id, data) - return data - - def delete_token(self, token_id): - return self.db.delete('token-%s' % token_id) - - -class KvsCatalog(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db - - # Public interface - def get_catalog(self, user_id, tenant_id, metadata=None): - return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) - - def get_service(self, service_id): - return self.db.get('service-%s' % service_id) - - def list_services(self): - return self.db.get('service_list', []) - - def create_service(self, service_id, service): - self.db.set('service-%s' % service_id, service) - service_list = set(self.db.get('service_list', [])) - service_list.add(service_id) - self.db.set('service_list', list(service_list)) - return service - - def update_service(self, service_id, service): - self.db.set('service-%s' % service_id, service) - return service - - def delete_service(self, service_id): - self.db.delete('service-%s' % service_id) - service_list = set(self.db.get('service_list', [])) - service_list.remove(service_id) - self.db.set('service_list', list(service_list)) - return None - - # Private interface - def _create_catalog(self, user_id, tenant_id, data): - self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) - return data - - -class KvsPolicy(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db - - def can_haz(self, target, action, credentials): - pass - - -class KvsEc2(object): - def __init__(self, db=None): - if db is None: - db = INMEMDB - elif type(db) is type({}): - db = DictKvs(db) - self.db = db - - # Public interface - def get_credential(self, credential_id): - credential_ref = self.db.get('credential-%s' % credential_id) - return credential_ref - - def list_credentials(self, user_id): - credential_ids = self.db.get('credential_list', []) - rv = [self.get_credential(x) for x in credential_ids] - return [x for x in rv if x['user_id'] == user_id] - - # CRUD - def create_credential(self, credential_id, credential): - self.db.set('credential-%s' % credential_id, credential) - credential_list = set(self.db.get('credential_list', [])) - credential_list.add(credential_id) - self.db.set('credential_list', list(credential_list)) - return credential - - def delete_credential(self, credential_id): - old_credential = self.db.get('credential-%s' % credential_id) - self.db.delete('credential-%s' % credential_id) - credential_list = set(self.db.get('credential_list', [])) - credential_list.remove(credential_id) - self.db.set('credential_list', list(credential_list)) - return None diff --git a/keystone/common/manager.py b/keystone/common/manager.py index d6d1f602d6..17c7d5e53b 100644 --- a/keystone/common/manager.py +++ b/keystone/common/manager.py @@ -3,7 +3,7 @@ import functools from keystone import config -from keystone import utils +from keystone.common import utils class Manager(object): diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 869141abdb..c3f19037b1 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -25,7 +25,7 @@ import subprocess import sys import urllib -from keystone import logging +from keystone.common import logging def import_class(import_str): diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index ee14970cb2..b7b244fa6f 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -81,7 +81,7 @@ class Request(webob.Request): pass -class Application(object): +class BaseApplication(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @classmethod @@ -146,6 +146,60 @@ class Application(object): raise NotImplementedError('You must implement __call__') +class Application(wsgi.BaseApplication): + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + del arg_dict['controller'] + logging.debug('arg_dict: %s', arg_dict) + + context = req.environ.get('openstack.context', {}) + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + params.update(arg_dict) + + # TODO(termie): do some basic normalization on methods + method = getattr(self, action) + + # NOTE(vish): make sure we have no unicode keys for py2.6. + params = self._normalize_dict(params) + result = method(context, **params) + + if result is None or type(result) is str or type(result) is unicode: + return result + elif isinstance(result, webob.exc.WSGIHTTPException): + return result + + return self._serialize(result) + + def _serialize(self, result): + return json.dumps(result, cls=utils.SmarterEncoder) + + def _normalize_arg(self, arg): + return str(arg).replace(':', '_').replace('-', '_') + + def _normalize_dict(self, d): + return dict([(self._normalize_arg(k), v) + for (k, v) in d.iteritems()]) + + def assert_admin(self, context): + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['metadata'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + print creds + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + + class Middleware(Application): """Base WSGI middleware. @@ -310,6 +364,31 @@ class Router(object): return app +class ComposingRouter(Router): + def __init__(self, mapper=None, routers=None): + if mapper is None: + mapper = routes.Mapper() + if routers is None: + routers = [] + for router in routers: + router.add_routes(mapper) + super(ComposingRouter, self).__init__(mapper) + + +class ComposableRouter(object): + """Router that supports use by ComposingRouter.""" + + def __init__(self, mapper=None): + if mapper is None: + mapper = routes.Mapper() + self.add_routes(mapper) + super(ComposableRouter, self).__init__(mapper) + + def add_routes(self, mapper): + """Add routes to given mapper.""" + pass + + class ExtensionRouter(Router): """A router that allows extensions to supplement or overwrite routes. @@ -317,10 +396,13 @@ class ExtensionRouter(Router): """ def __init__(self, application, mapper): self.application = application - + self.add_routes(mapper) mapper.connect('{path_info:.*}', controller=self.application) super(ExtensionRouter, self).__init__(mapper) + def add_routes(self, mapper): + pass + @classmethod def factory(cls, global_config, **local_config): """Used for paste app factories in paste.deploy config files. diff --git a/keystone/contrib/admin_crud/__init__.py b/keystone/contrib/admin_crud/__init__.py index e69de29bb2..eb8c4f17e6 100644 --- a/keystone/contrib/admin_crud/__init__.py +++ b/keystone/contrib/admin_crud/__init__.py @@ -0,0 +1 @@ +from keystone.contrib.admin_crud.core import * diff --git a/keystone/contrib/admin_crud/core.py b/keystone/contrib/admin_crud/core.py new file mode 100644 index 0000000000..f58b759e63 --- /dev/null +++ b/keystone/contrib/admin_crud/core.py @@ -0,0 +1,150 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone import catalog +from keystone import identity +from keystone.common import wsgi + + +class CrudExtension(wsgi.ExtensionRouter): + """Previously known as the OS-KSADM extension. + + Provides a bunch of CRUD operations for internal data types. + + """ + + def add_routes(self, mapper): + tenant_controller = identity.TenantController() + user_controller = identity.UserController() + role_controller = identity.RoleController() + service_controller = catalog.ServiceController() + + # Tenant Operations + mapper.connect("/tenants", controller=tenant_controller, + action="create_tenant", + conditions=dict(method=["POST"])) + mapper.connect("/tenants/{tenant_id}", + controller=tenant_controller, + action="update_tenant", + conditions=dict(method=["PUT"])) + mapper.connect("/tenants/{tenant_id}", + controller=tenant_controller, + action="delete_tenant", + conditions=dict(method=["DELETE"])) + mapper.connect("/tenants/{tenant_id}/users", + controller=user_controller, + action="get_tenant_users", + conditions=dict(method=["GET"])) + + # User Operations + mapper.connect("/users", + controller=user_controller, + action="get_users", + conditions=dict(method=["GET"])) + mapper.connect("/users", + controller=user_controller, + action="create_user", + conditions=dict(method=["POST"])) + # NOTE(termie): not in diablo + mapper.connect("/users/{user_id}", + controller=user_controller, + action="update_user", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}", + controller=user_controller, + action="delete_user", + conditions=dict(method=["DELETE"])) + + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/password", + controller=user_controller, + action="set_user_password", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/password", + controller=user_controller, + action="set_user_password", + conditions=dict(method=["PUT"])) + + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/tenant", + controller=user_controller, + action="update_user_tenant", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/tenant", + controller=user_controller, + action="update_user_tenant", + conditions=dict(method=["PUT"])) + + # COMPAT(diablo): the copy with no OS-KSADM is from diablo + mapper.connect("/users/{user_id}/enabled", + controller=user_controller, + action="set_user_enabled", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/OS-KSADM/enabled", + controller=user_controller, + action="set_user_enabled", + conditions=dict(method=["PUT"])) + + # User Roles + mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=role_controller, action="add_role_to_user", + conditions=dict(method=["PUT"])) + mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=role_controller, action="delete_role_from_user", + conditions=dict(method=["DELETE"])) + + # COMPAT(diablo): User Roles + mapper.connect("/users/{user_id}/roleRefs", + controller=role_controller, action="get_role_refs", + conditions=dict(method=["GET"])) + mapper.connect("/users/{user_id}/roleRefs", + controller=role_controller, action="create_role_ref", + conditions=dict(method=["POST"])) + mapper.connect("/users/{user_id}/roleRefs/{role_ref_id}", + controller=role_controller, action="delete_role_ref", + conditions=dict(method=["DELETE"])) + + # User-Tenant Roles + mapper.connect( + "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=role_controller, action="add_role_to_user", + conditions=dict(method=["PUT"])) + mapper.connect( + "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", + controller=role_controller, action="delete_role_from_user", + conditions=dict(method=["DELETE"])) + + # Service Operations + mapper.connect("/OS-KSADM/services", + controller=service_controller, + action="get_services", + conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/services", + controller=service_controller, + action="create_service", + conditions=dict(method=["POST"])) + mapper.connect("/OS-KSADM/services/{service_id}", + controller=service_controller, + action="delete_service", + conditions=dict(method=["DELETE"])) + mapper.connect("/OS-KSADM/services/{service_id}", + controller=service_controller, + action="get_service", + conditions=dict(method=["GET"])) + + # Role Operations + mapper.connect("/OS-KSADM/roles", + controller=role_controller, + action="create_role", + conditions=dict(method=["POST"])) + mapper.connect("/OS-KSADM/roles", + controller=role_controller, + action="get_roles", + conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/roles/{role_id}", + controller=role_controller, + action="get_role", + conditions=dict(method=["GET"])) + mapper.connect("/OS-KSADM/roles/{role_id}", + controller=role_controller, + action="delete_role", + conditions=dict(method=["DELETE"])) diff --git a/keystone/contrib/ec2/__init__.py b/keystone/contrib/ec2/__init__.py index e69de29bb2..6c85c66df7 100644 --- a/keystone/contrib/ec2/__init__.py +++ b/keystone/contrib/ec2/__init__.py @@ -0,0 +1 @@ +from keystone.contrib.ec2.core import * diff --git a/keystone/contrib/ec2/backends/__init__.py b/keystone/contrib/ec2/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/contrib/ec2/backends/kvs.py b/keystone/contrib/ec2/backends/kvs.py new file mode 100644 index 0000000000..47975ed5cb --- /dev/null +++ b/keystone/contrib/ec2/backends/kvs.py @@ -0,0 +1,32 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone.common import kvs + + +class Ec2(kvs.Base): + # Public interface + def get_credential(self, credential_id): + credential_ref = self.db.get('credential-%s' % credential_id) + return credential_ref + + def list_credentials(self, user_id): + credential_ids = self.db.get('credential_list', []) + rv = [self.get_credential(x) for x in credential_ids] + return [x for x in rv if x['user_id'] == user_id] + + # CRUD + def create_credential(self, credential_id, credential): + self.db.set('credential-%s' % credential_id, credential) + credential_list = set(self.db.get('credential_list', [])) + credential_list.add(credential_id) + self.db.set('credential_list', list(credential_list)) + return credential + + def delete_credential(self, credential_id): + old_credential = self.db.get('credential-%s' % credential_id) + self.db.delete('credential-%s' % credential_id) + credential_list = set(self.db.get('credential_list', [])) + credential_list.remove(credential_id) + self.db.set('credential_list', list(credential_list)) + return None + diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index a02e4daf22..1c83689c00 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -2,8 +2,13 @@ """Main entry point into the EC2 Credentials service.""" +from keystone import catalog from keystone import config -from keystone import manager +from keystone import identity +from keystone import policy +from keystone import token +from keystone.common import manager +from keystone.common import wsgi CONF = config.CONF @@ -22,3 +27,131 @@ class Manager(manager.Manager): def __init__(self): super(Manager, self).__init__(CONF.ec2.driver) + + +class Ec2Extension(wsgi.ExtensionRouter): + def add_routes(self, mapper) + ec2_controller = Ec2Controller() + # validation + mapper.connect('/ec2tokens', + controller=ec2_controller, + action='authenticate_ec2', + conditions=dict(method=['POST'])) + + # crud + mapper.connect('/users/{user_id}/credentials/OS-EC2', + controller=ec2_controller, + action='create_credential', + conditions=dict(method=['POST'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2', + controller=ec2_controller, + action='get_credentials', + conditions=dict(method=['GET'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', + controller=ec2_controller, + action='get_credential', + conditions=dict(method=['GET'])) + mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', + controller=ec2_controller, + action='delete_credential', + conditions=dict(method=['DELETE'])) + + +class Ec2Controller(wsgi.Application): + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() + self.ec2_api = Manager() + super(Ec2Controller, self).__init__() + + def authenticate_ec2(self, context, credentials=None, + ec2Credentials=None): + """Validate a signed EC2 request and provide a token.""" + # NOTE(termie): backwards compat hack + if not credentials and ec2Credentials: + credentials = ec2Credentials + creds_ref = self.ec2_api.get_credential(context, + credentials['access']) + + signer = utils.Signer(creds_ref['secret']) + signature = signer.generate(credentials) + if signature == credentials['signature']: + pass + # NOTE(vish): Some libraries don't use the port when signing + # requests, so try again without port. + elif ':' in credentials['signature']: + hostname, _port = credentials['host'].split(":") + credentials['host'] = hostname + signature = signer.generate(credentials) + if signature != credentials.signature: + # TODO(termie): proper exception + raise Exception("Not Authorized") + else: + raise Exception("Not Authorized") + + # TODO(termie): don't create new tokens every time + # TODO(termie): this is copied from TokenController.authenticate + token_id = uuid.uuid4().hex + tenant_ref = self.identity_api.get_tenant(creds_ref['tenant_id']) + user_ref = self.identity_api.get_user(creds_ref['user_id']) + metadata_ref = self.identity_api.get_metadata( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id']) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + metadata=metadata_ref) + + token_ref = self.token_api.create_token( + context, token_id, dict(expires='', + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) + + # TODO(termie): optimize this call at some point and put it into the + # the return for metadata + # fill out the roles in the metadata + roles_ref = [] + for role_id in metadata_ref.get('roles', []): + roles_ref.append(self.identity_api.get_role(context, role_id)) + + # TODO(termie): make this a util function or something + # TODO(termie): i don't think the ec2 middleware currently expects a + # full return, but it contains a note saying that it + # would be better to expect a full return + return TokenController._format_authenticate( + self, token_ref, roles_ref, catalog_ref) + + def create_credential(self, context, user_id, tenant_id): + # TODO(termie): validate that this request is valid for given user + # tenant + cred_ref = {'user_id': user_id, + 'tenant_id': tenant_id, + 'access': uuid.uuid4().hex, + 'secret': uuid.uuid4().hex} + self.ec2_api.create_credential(context, cred_ref['access'], cred_ref) + return {'credential': cred_ref} + + def get_credentials(self, context, user_id): + """List credentials for the given user_id.""" + # TODO(termie): validate that this request is valid for given user + # tenant + return {'credentials': self.ec2_api.list_credentials(context, user_id)} + + def get_credential(self, context, user_id, credential_id): + # TODO(termie): validate that this request is valid for given user + # tenant + return {'credential': self.ec2_api.get_credential(context, + credential_id)} + + def delete_credential(self, context, user_id, credential_id): + # TODO(termie): validate that this request is valid for given user + # tenant + return self.ec2_api.delete_credential(context, credential_id) + + diff --git a/keystone/identity/__init__.py b/keystone/identity/__init__.py index e69de29bb2..3a86d5a512 100644 --- a/keystone/identity/__init__.py +++ b/keystone/identity/__init__.py @@ -0,0 +1 @@ +from keystone.identity.core import * diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py new file mode 100644 index 0000000000..233884c62e --- /dev/null +++ b/keystone/identity/backends/kvs.py @@ -0,0 +1,178 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone.common import kvs + + +class Identity(kvs.Base): + # Public interface + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. + + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. + + """ + user_ref = self.get_user(user_id) + tenant_ref = None + metadata_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') + if tenant_id and tenant_id not in user_ref['tenants']: + raise AssertionError('Invalid tenant') + + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + metadata_ref = self.get_metadata(user_id, tenant_id) + else: + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) + + def get_tenant(self, tenant_id): + tenant_ref = self.db.get('tenant-%s' % tenant_id) + return tenant_ref + + def get_tenant_by_name(self, tenant_name): + tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + return tenant_ref + + def get_user(self, user_id): + user_ref = self.db.get('user-%s' % user_id) + return user_ref + + def get_user_by_name(self, user_name): + user_ref = self.db.get('user_name-%s' % user_name) + return user_ref + + def get_metadata(self, user_id, tenant_id): + return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) + + def get_role(self, role_id): + role_ref = self.db.get('role-%s' % role_id) + return role_ref + + def list_users(self): + user_ids = self.db.get('user_list', []) + return [self.get_user(x) for x in user_ids] + + def list_roles(self): + role_ids = self.db.get('role_list', []) + return [self.get_role(x) for x in role_ids] + + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.add(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def remove_user_from_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.remove(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def get_tenants_for_user(self, user_id): + user_ref = self.get_user(user_id) + return user_ref.get('tenants', []) + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) + + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.add(role_id) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.remove(role_id) + metadata_ref['roles'] = list(roles) + self.update_metadata(user_id, tenant_id, metadata_ref) + + # CRUD + def create_user(self, user_id, user): + self.db.set('user-%s' % user_id, user) + self.db.set('user_name-%s' % user['name'], user) + user_list = set(self.db.get('user_list', [])) + user_list.add(user_id) + self.db.set('user_list', list(user_list)) + return user + + def update_user(self, user_id, user): + # get the old name and delete it too + old_user = self.db.get('user-%s' % user_id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.set('user-%s' % user_id, user) + self.db.set('user_name-%s' % user['name'], user) + return user + + def delete_user(self, user_id): + old_user = self.db.get('user-%s' % user_id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.delete('user-%s' % user_id) + user_list = set(self.db.get('user_list', [])) + user_list.remove(user_id) + self.db.set('user_list', list(user_list)) + return None + + def create_tenant(self, tenant_id, tenant): + self.db.set('tenant-%s' % tenant_id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def update_tenant(self, tenant_id, tenant): + # get the old name and delete it too + old_tenant = self.db.get('tenant-%s' % tenant_id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.set('tenant-%s' % tenant_id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def delete_tenant(self, tenant_id): + old_tenant = self.db.get('tenant-%s' % tenant_id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.delete('tenant-%s' % tenant_id) + return None + + def create_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata + + def update_metadata(self, user_id, tenant_id, metadata): + self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata) + return metadata + + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) + return None + + def create_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) + role_list = set(self.db.get('role_list', [])) + role_list.add(role_id) + self.db.set('role_list', list(role_list)) + return role + + def update_role(self, role_id, role): + self.db.set('role-%s' % role_id, role) + return role + + def delete_role(self, role_id): + self.db.delete('role-%s' % role_id) + role_list = set(self.db.get('role_list', [])) + role_list.remove(role_id) + self.db.set('role_list', list(role_list)) + return None + diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 5f25fbf47b..4400d7f6eb 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -22,3 +22,296 @@ class Manager(manager.Manager): def __init__(self): super(Manager, self).__init__(CONF.identity.driver) + + +class PublicRouter(wsgi.ComposableRouter): + def add_routes(self, mapper): + tenant_controller = TenantController() + mapper.connect('/tenants', + controller=tenant_controller, + action='get_tenants_for_token', + conditions=dict(methods=['GET'])) + + +class AdminRouter(wsgi.ComposableRouter): + def add_routes(self, mapper): + # Tenant Operations + tenant_controller = TenantController() + mapper.connect('/tenants', + controller=tenant_controller, + action='get_tenants_for_token', + conditions=dict(method=['GET'])) + mapper.connect('/tenants/{tenant_id}', + controller=tenant_controller, + action='get_tenant', + conditions=dict(method=['GET'])) + + # User Operations + user_controller = UserController() + mapper.connect('/users/{user_id}', + controller=user_controller, + action='get_user', + conditions=dict(method=['GET'])) + + # Role Operations + roles_controller = RoleController() + mapper.connect('/tenants/{tenant_id}/users/{user_id}/roles', + controller=roles_controller, + action='get_user_roles', + conditions=dict(method=['GET'])) + mapper.connect('/users/{user_id}/roles', + controller=user_controller, + action='get_user_roles', + conditions=dict(method=['GET'])) + + +class TenantController(Application): + def __init__(self): + self.identity_api = identity.Manager() + self.policy_api = policy.Manager() + self.token_api = token.Manager() + super(TenantController, self).__init__() + + def get_tenants_for_token(self, context, **kw): + """Get valid tenants for token based on token used to authenticate. + + Pulls the token from the context, validates it and gets the valid + tenants for the user in the token. + + Doesn't care about token scopedness. + + """ + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + assert token_ref is not None + + user_ref = token_ref['user'] + tenant_ids = self.identity_api.get_tenants_for_user( + context, user_ref['id']) + tenant_refs = [] + for tenant_id in tenant_ids: + tenant_refs.append(self.identity_api.get_tenant( + context=context, + tenant_id=tenant_id)) + return self._format_tenants_for_token(tenant_refs) + + def get_tenant(self, context, tenant_id): + # TODO(termie): this stuff should probably be moved to middleware + if not context['is_admin']: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + creds = user_token_ref['metadata'].copy() + creds['user_id'] = user_token_ref['user'].get('id') + creds['tenant_id'] = user_token_ref['tenant'].get('id') + # Accept either is_admin or the admin role + assert self.policy_api.can_haz(context, + ('is_admin:1', 'roles:admin'), + creds) + + tenant = self.identity_api.get_tenant(context, tenant_id) + if not tenant: + return webob.exc.HTTPNotFound() + return {'tenant': tenant} + + # CRUD Extension + def create_tenant(self, context, tenant): + tenant_ref = self._normalize_dict(tenant) + self.assert_admin(context) + tenant_id = (tenant_ref.get('id') + and tenant_ref.get('id') + or uuid.uuid4().hex) + tenant_ref['id'] = tenant_id + + tenant = self.identity_api.create_tenant( + context, tenant_id, tenant_ref) + return {'tenant': tenant} + + def update_tenant(self, context, tenant_id, tenant): + self.assert_admin(context) + tenant_ref = self.identity_api.update_tenant( + context, tenant_id, tenant) + return {'tenant': tenant_ref} + + def delete_tenant(self, context, tenant_id, **kw): + self.assert_admin(context) + self.identity_api.delete_tenant(context, tenant_id) + + def get_tenant_users(self, context, **kw): + self.assert_admin(context) + raise NotImplementedError() + + def _format_tenants_for_token(self, tenant_refs): + for x in tenant_refs: + x['enabled'] = True + o = {'tenants': tenant_refs, + 'tenants_links': []} + return o + + +class UserController(Application): + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.policy_api = policy.Manager() + self.token_api = token.Manager() + super(UserController, self).__init__() + + def get_user(self, context, user_id): + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + if not user_ref: + raise webob.exc.HTTPNotFound() + return {'user': user_ref} + + def get_users(self, context): + # NOTE(termie): i can't imagine that this really wants all the data + # about every single user in the system... + self.assert_admin(context) + user_refs = self.identity_api.list_users(context) + return {'users': user_refs} + + # CRUD extension + def create_user(self, context, user): + user = self._normalize_dict(user) + self.assert_admin(context) + tenant_id = user.get('tenantId', None) + user_id = uuid.uuid4().hex + user_ref = user.copy() + user_ref['id'] = user_id + new_user_ref = self.identity_api.create_user( + context, user_id, user_ref) + if tenant_id: + self.identity_api.add_user_to_tenant(tenant_id, user_id) + return {'user': new_user_ref} + + # NOTE(termie): this is really more of a patch than a put + def update_user(self, context, user_id, user): + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + del user['id'] + user_ref.update(user) + self.identity_api.update_user(context, user_id, user_ref) + return {'user': user_ref} + + def delete_user(self, context, user_id): + self.assert_admin(context) + self.identity_api.delete_user(context, user_id) + + def set_user_enabled(self, context, user_id, user): + return self.update_user(context, user_id, user) + + def set_user_password(self, context, user_id, user): + return self.update_user(context, user_id, user) + + def update_user_tenant(self, context, user_id, user): + """Update the default tenant.""" + # ensure that we're a member of that tenant + tenant_id = user.get('tenantId') + self.identity_api.add_user_to_tenant(context, tenant_id, user_id) + return self.update_user(context, user_id, user) + + +class RoleController(Application): + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() + super(RoleController, self).__init__() + + def get_user_roles(self, context, user_id, tenant_id=None): + raise NotImplemented() + + # CRUD extension + def get_role(self, context, role_id): + self.assert_admin(context) + role_ref = self.identity_api.get_role(context, role_id) + if not role_ref: + raise webob.exc.HTTPNotFound() + return {'role': role_ref} + + def create_role(self, context, role): + role = self._normalize_dict(role) + self.assert_admin(context) + role_id = uuid.uuid4().hex + role['id'] = role_id + role_ref = self.identity_api.create_role(context, role_id, role) + return {'role': role_ref} + + def delete_role(self, context, role_id): + self.assert_admin(context) + role_ref = self.identity_api.delete_role(context, role_id) + + def get_roles(self, context): + self.assert_admin(context) + roles = self.identity_api.list_roles(context) + # TODO(termie): probably inefficient at some point + return {'roles': roles} + + # COMPAT(diablo): CRUD extension + def get_role_refs(self, context, user_id): + """Ultimate hack to get around having to make role_refs first-class. + + This will basically iterate over the various roles the user has in + all tenants the user is a member of and create fake role_refs where + the id encodes the user-tenant-role information so we can look + up the appropriate data when we need to delete them. + + """ + self.assert_admin(context) + user_ref = self.identity_api.get_user(context, user_id) + tenant_ids = self.identity_api.get_tenants_for_user(context, user_id) + o = [] + for tenant_id in tenant_ids: + role_ids = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + for role_id in role_ids: + ref = {'roleId': role_id, + 'tenantId': tenant_id, + 'userId': user_id} + ref['id'] = urllib.urlencode(ref) + o.append(ref) + return {'roles': o} + + def create_role_ref(self, context, user_id, role): + """This is actually used for adding a user to a tenant. + + In the legacy data model adding a user to a tenant required setting + a role. + + """ + self.assert_admin(context) + # TODO(termie): for now we're ignoring the actual role + tenant_id = role.get('tenantId') + role_id = role.get('roleId') + self.identity_api.add_user_to_tenant(context, tenant_id, user_id) + self.identity_api.add_role_to_user_and_tenant( + context, user_id, tenant_id, role_id) + role_ref = self.identity_api.get_role(context, role_id) + return {'role': role_ref} + + def delete_role_ref(self, context, user_id, role_ref_id): + """This is actually used for deleting a user from a tenant. + + In the legacy data model removing a user from a tenant required + deleting a role. + + To emulate this, we encode the tenant and role in the role_ref_id, + and if this happens to be the last role for the user-tenant pair, + we remove the user from the tenant. + + """ + self.assert_admin(context) + # TODO(termie): for now we're ignoring the actual role + role_ref_ref = urlparse.parse_qs(role_ref_id) + tenant_id = role_ref_ref.get('tenantId')[0] + role_id = role_ref_ref.get('roleId')[0] + self.identity_api.remove_role_from_user_and_tenant( + context, user_id, tenant_id, role_id) + roles = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + if not roles: + self.identity_api.remove_user_from_tenant( + context, tenant_id, user_id) + + diff --git a/keystone/policy/backends/simple.py b/keystone/policy/backends/simple.py index 643d3e2136..206ab574b1 100644 --- a/keystone/policy/backends/simple.py +++ b/keystone/policy/backends/simple.py @@ -1,7 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -import logging +from keystone.common import logging class TrivialTrue(object): diff --git a/keystone/service.py b/keystone/service.py index 23c66bc1b8..bdb5318199 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -10,70 +10,15 @@ import webob.dec import webob.exc from keystone import catalog -from keystone import ec2 from keystone import identity -from keystone import logging from keystone import policy from keystone import token -from keystone import utils -from keystone import wsgi +from keystone.common import logging +from keystone.common import utils +from keystone.common import wsgi -class Application(wsgi.Application): - @webob.dec.wsgify - def __call__(self, req): - arg_dict = req.environ['wsgiorg.routing_args'][1] - action = arg_dict['action'] - del arg_dict['action'] - del arg_dict['controller'] - logging.debug('arg_dict: %s', arg_dict) - - context = req.environ.get('openstack.context', {}) - # allow middleware up the stack to override the params - params = {} - if 'openstack.params' in req.environ: - params = req.environ['openstack.params'] - params.update(arg_dict) - - # TODO(termie): do some basic normalization on methods - method = getattr(self, action) - - # NOTE(vish): make sure we have no unicode keys for py2.6. - params = self._normalize_dict(params) - result = method(context, **params) - - if result is None or type(result) is str or type(result) is unicode: - return result - elif isinstance(result, webob.exc.WSGIHTTPException): - return result - - return self._serialize(result) - - def _serialize(self, result): - return json.dumps(result, cls=utils.SmarterEncoder) - - def _normalize_arg(self, arg): - return str(arg).replace(':', '_').replace('-', '_') - - def _normalize_dict(self, d): - return dict([(self._normalize_arg(k), v) - for (k, v) in d.iteritems()]) - - def assert_admin(self, context): - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['metadata'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - print creds - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) - - -class AdminRouter(wsgi.Router): +class AdminRouter(wsgi.ComposingRouter): def __init__(self): mapper = routes.Mapper() @@ -92,34 +37,6 @@ class AdminRouter(wsgi.Router): action='endpoints', conditions=dict(method=['GET'])) - # Tenant Operations - tenant_controller = TenantController() - mapper.connect('/tenants', - controller=tenant_controller, - action='get_tenants_for_token', - conditions=dict(method=['GET'])) - mapper.connect('/tenants/{tenant_id}', - controller=tenant_controller, - action='get_tenant', - conditions=dict(method=['GET'])) - - # User Operations - user_controller = UserController() - mapper.connect('/users/{user_id}', - controller=user_controller, - action='get_user', - conditions=dict(method=['GET'])) - - # Role Operations - roles_controller = RoleController() - mapper.connect('/tenants/{tenant_id}/users/{user_id}/roles', - controller=roles_controller, - action='get_user_roles', - conditions=dict(method=['GET'])) - mapper.connect('/users/{user_id}/roles', - controller=user_controller, - action='get_user_roles', - conditions=dict(method=['GET'])) # Miscellaneous Operations version_controller = VersionController() @@ -133,11 +50,12 @@ class AdminRouter(wsgi.Router): controller=extensions_controller, action='get_extensions_info', conditions=dict(method=['GET'])) - - super(AdminRouter, self).__init__(mapper) + identity_router = identity.AdminRouter() + routers = [identity_router] + super(AdminRouter, self).__init__(mapper, identity_router) -class PublicRouter(wsgi.Router): +class PublicRouter(wsgi.ComposingRouter): def __init__(self): mapper = routes.Mapper() @@ -153,13 +71,6 @@ class PublicRouter(wsgi.Router): action='authenticate', conditions=dict(method=['POST'])) - # Tenant Operations - tenant_controller = TenantController() - mapper.connect('/tenants', - controller=tenant_controller, - action='get_tenants_for_token', - conditions=dict(methods=['GET'])) - # Miscellaneous version_controller = VersionController() mapper.connect('/', @@ -174,289 +85,13 @@ class PublicRouter(wsgi.Router): action='get_extensions_info', conditions=dict(method=['GET'])) - super(PublicRouter, self).__init__(mapper) + identity_router = identity.PublicRouter() + routers = [identity_router] + + super(PublicRouter, self).__init__(mapper, routers) -class AdminCrudExtension(wsgi.ExtensionRouter): - """Previously known as the OS-KSADM extension. - - Provides a bunch of CRUD operations for internal data types. - - """ - - def __init__(self, application): - mapper = routes.Mapper() - tenant_controller = TenantController() - user_controller = UserController() - role_controller = RoleController() - service_controller = ServiceController() - - # Tenant Operations - mapper.connect("/tenants", controller=tenant_controller, - action="create_tenant", - conditions=dict(method=["POST"])) - mapper.connect("/tenants/{tenant_id}", - controller=tenant_controller, - action="update_tenant", - conditions=dict(method=["PUT"])) - mapper.connect("/tenants/{tenant_id}", - controller=tenant_controller, - action="delete_tenant", - conditions=dict(method=["DELETE"])) - mapper.connect("/tenants/{tenant_id}/users", - controller=user_controller, - action="get_tenant_users", - conditions=dict(method=["GET"])) - - # User Operations - mapper.connect("/users", - controller=user_controller, - action="get_users", - conditions=dict(method=["GET"])) - mapper.connect("/users", - controller=user_controller, - action="create_user", - conditions=dict(method=["POST"])) - # NOTE(termie): not in diablo - mapper.connect("/users/{user_id}", - controller=user_controller, - action="update_user", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}", - controller=user_controller, - action="delete_user", - conditions=dict(method=["DELETE"])) - - # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/password", - controller=user_controller, - action="set_user_password", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/password", - controller=user_controller, - action="set_user_password", - conditions=dict(method=["PUT"])) - - # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/tenant", - controller=user_controller, - action="update_user_tenant", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/tenant", - controller=user_controller, - action="update_user_tenant", - conditions=dict(method=["PUT"])) - - # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/enabled", - controller=user_controller, - action="set_user_enabled", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/enabled", - controller=user_controller, - action="set_user_enabled", - conditions=dict(method=["PUT"])) - - # User Roles - mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="add_role_to_user", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="delete_role_from_user", - conditions=dict(method=["DELETE"])) - - # COMPAT(diablo): User Roles - mapper.connect("/users/{user_id}/roleRefs", - controller=role_controller, action="get_role_refs", - conditions=dict(method=["GET"])) - mapper.connect("/users/{user_id}/roleRefs", - controller=role_controller, action="create_role_ref", - conditions=dict(method=["POST"])) - mapper.connect("/users/{user_id}/roleRefs/{role_ref_id}", - controller=role_controller, action="delete_role_ref", - conditions=dict(method=["DELETE"])) - - # User-Tenant Roles - mapper.connect( - "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="add_role_to_user", - conditions=dict(method=["PUT"])) - mapper.connect( - "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="delete_role_from_user", - conditions=dict(method=["DELETE"])) - - # Service Operations - mapper.connect("/OS-KSADM/services", - controller=service_controller, - action="get_services", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/services", - controller=service_controller, - action="create_service", - conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/services/{service_id}", - controller=service_controller, - action="delete_service", - conditions=dict(method=["DELETE"])) - mapper.connect("/OS-KSADM/services/{service_id}", - controller=service_controller, - action="get_service", - conditions=dict(method=["GET"])) - - # Role Operations - mapper.connect("/OS-KSADM/roles", - controller=role_controller, - action="create_role", - conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/roles", - controller=role_controller, - action="get_roles", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/roles/{role_id}", - controller=role_controller, - action="get_role", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/roles/{role_id}", - controller=role_controller, - action="delete_role", - conditions=dict(method=["DELETE"])) - - super(AdminCrudExtension, self).__init__( - application, mapper) - - -class Ec2Extension(wsgi.ExtensionRouter): - def __init__(self, application): - mapper = routes.Mapper() - ec2_controller = Ec2Controller() - - # validation - mapper.connect('/ec2tokens', - controller=ec2_controller, - action='authenticate_ec2', - conditions=dict(method=['POST'])) - - # crud - mapper.connect('/users/{user_id}/credentials/OS-EC2', - controller=ec2_controller, - action='create_credential', - conditions=dict(method=['POST'])) - mapper.connect('/users/{user_id}/credentials/OS-EC2', - controller=ec2_controller, - action='get_credentials', - conditions=dict(method=['GET'])) - mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', - controller=ec2_controller, - action='get_credential', - conditions=dict(method=['GET'])) - mapper.connect('/users/{user_id}/credentials/OS-EC2/{credential_id}', - controller=ec2_controller, - action='delete_credential', - conditions=dict(method=['DELETE'])) - - super(Ec2Extension, self).__init__(application, mapper) - - -class Ec2Controller(Application): - def __init__(self): - self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() - self.token_api = token.Manager() - self.policy_api = policy.Manager() - self.ec2_api = ec2.Manager() - super(Ec2Controller, self).__init__() - - def authenticate_ec2(self, context, credentials=None, - ec2Credentials=None): - """Validate a signed EC2 request and provide a token.""" - # NOTE(termie): backwards compat hack - if not credentials and ec2Credentials: - credentials = ec2Credentials - creds_ref = self.ec2_api.get_credential(context, - credentials['access']) - - signer = utils.Signer(creds_ref['secret']) - signature = signer.generate(credentials) - if signature == credentials['signature']: - pass - # NOTE(vish): Some libraries don't use the port when signing - # requests, so try again without port. - elif ':' in credentials['signature']: - hostname, _port = credentials['host'].split(":") - credentials['host'] = hostname - signature = signer.generate(credentials) - if signature != credentials.signature: - # TODO(termie): proper exception - raise Exception("Not Authorized") - else: - raise Exception("Not Authorized") - - # TODO(termie): don't create new tokens every time - # TODO(termie): this is copied from TokenController.authenticate - token_id = uuid.uuid4().hex - tenant_ref = self.identity_api.get_tenant(creds_ref['tenant_id']) - user_ref = self.identity_api.get_user(creds_ref['user_id']) - metadata_ref = self.identity_api.get_metadata( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id']) - catalog_ref = self.catalog_api.get_catalog( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id'], - metadata=metadata_ref) - - token_ref = self.token_api.create_token( - context, token_id, dict(expires='', - id=token_id, - user=user_ref, - tenant=tenant_ref, - metadata=metadata_ref)) - - # TODO(termie): optimize this call at some point and put it into the - # the return for metadata - # fill out the roles in the metadata - roles_ref = [] - for role_id in metadata_ref.get('roles', []): - roles_ref.append(self.identity_api.get_role(context, role_id)) - - # TODO(termie): make this a util function or something - # TODO(termie): i don't think the ec2 middleware currently expects a - # full return, but it contains a note saying that it - # would be better to expect a full return - return TokenController._format_authenticate( - self, token_ref, roles_ref, catalog_ref) - - def create_credential(self, context, user_id, tenant_id): - # TODO(termie): validate that this request is valid for given user - # tenant - cred_ref = {'user_id': user_id, - 'tenant_id': tenant_id, - 'access': uuid.uuid4().hex, - 'secret': uuid.uuid4().hex} - self.ec2_api.create_credential(context, cred_ref['access'], cred_ref) - return {'credential': cred_ref} - - def get_credentials(self, context, user_id): - """List credentials for the given user_id.""" - # TODO(termie): validate that this request is valid for given user - # tenant - return {'credentials': self.ec2_api.list_credentials(context, user_id)} - - def get_credential(self, context, user_id, credential_id): - # TODO(termie): validate that this request is valid for given user - # tenant - return {'credential': self.ec2_api.get_credential(context, - credential_id)} - - def delete_credential(self, context, user_id, credential_id): - # TODO(termie): validate that this request is valid for given user - # tenant - return self.ec2_api.delete_credential(context, credential_id) - - -class NoopController(Application): +class NoopController(wsgi.Application): def __init__(self): super(NoopController, self).__init__() @@ -464,7 +99,7 @@ class NoopController(Application): return {} -class TokenController(Application): +class TokenController(wsgi.Application): def __init__(self): self.catalog_api = catalog.Manager() self.identity_api = identity.Manager() @@ -693,291 +328,7 @@ class TokenController(Application): return services.values() -class TenantController(Application): - def __init__(self): - self.identity_api = identity.Manager() - self.policy_api = policy.Manager() - self.token_api = token.Manager() - super(TenantController, self).__init__() - - def get_tenants_for_token(self, context, **kw): - """Get valid tenants for token based on token used to authenticate. - - Pulls the token from the context, validates it and gets the valid - tenants for the user in the token. - - Doesn't care about token scopedness. - - """ - token_ref = self.token_api.get_token(context=context, - token_id=context['token_id']) - assert token_ref is not None - - user_ref = token_ref['user'] - tenant_ids = self.identity_api.get_tenants_for_user( - context, user_ref['id']) - tenant_refs = [] - for tenant_id in tenant_ids: - tenant_refs.append(self.identity_api.get_tenant( - context=context, - tenant_id=tenant_id)) - return self._format_tenants_for_token(tenant_refs) - - def get_tenant(self, context, tenant_id): - # TODO(termie): this stuff should probably be moved to middleware - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['metadata'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) - - tenant = self.identity_api.get_tenant(context, tenant_id) - if not tenant: - return webob.exc.HTTPNotFound() - return {'tenant': tenant} - - # CRUD Extension - def create_tenant(self, context, tenant): - tenant_ref = self._normalize_dict(tenant) - self.assert_admin(context) - tenant_id = (tenant_ref.get('id') - and tenant_ref.get('id') - or uuid.uuid4().hex) - tenant_ref['id'] = tenant_id - - tenant = self.identity_api.create_tenant( - context, tenant_id, tenant_ref) - return {'tenant': tenant} - - def update_tenant(self, context, tenant_id, tenant): - self.assert_admin(context) - tenant_ref = self.identity_api.update_tenant( - context, tenant_id, tenant) - return {'tenant': tenant_ref} - - def delete_tenant(self, context, tenant_id, **kw): - self.assert_admin(context) - self.identity_api.delete_tenant(context, tenant_id) - - def get_tenant_users(self, context, **kw): - self.assert_admin(context) - raise NotImplementedError() - - def _format_tenants_for_token(self, tenant_refs): - for x in tenant_refs: - x['enabled'] = True - o = {'tenants': tenant_refs, - 'tenants_links': []} - return o - - -class UserController(Application): - def __init__(self): - self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() - self.policy_api = policy.Manager() - self.token_api = token.Manager() - super(UserController, self).__init__() - - def get_user(self, context, user_id): - self.assert_admin(context) - user_ref = self.identity_api.get_user(context, user_id) - if not user_ref: - raise webob.exc.HTTPNotFound() - return {'user': user_ref} - - def get_users(self, context): - # NOTE(termie): i can't imagine that this really wants all the data - # about every single user in the system... - self.assert_admin(context) - user_refs = self.identity_api.list_users(context) - return {'users': user_refs} - - # CRUD extension - def create_user(self, context, user): - user = self._normalize_dict(user) - self.assert_admin(context) - tenant_id = user.get('tenantId', None) - user_id = uuid.uuid4().hex - user_ref = user.copy() - user_ref['id'] = user_id - new_user_ref = self.identity_api.create_user( - context, user_id, user_ref) - if tenant_id: - self.identity_api.add_user_to_tenant(tenant_id, user_id) - return {'user': new_user_ref} - - # NOTE(termie): this is really more of a patch than a put - def update_user(self, context, user_id, user): - self.assert_admin(context) - user_ref = self.identity_api.get_user(context, user_id) - del user['id'] - user_ref.update(user) - self.identity_api.update_user(context, user_id, user_ref) - return {'user': user_ref} - - def delete_user(self, context, user_id): - self.assert_admin(context) - self.identity_api.delete_user(context, user_id) - - def set_user_enabled(self, context, user_id, user): - return self.update_user(context, user_id, user) - - def set_user_password(self, context, user_id, user): - return self.update_user(context, user_id, user) - - def update_user_tenant(self, context, user_id, user): - """Update the default tenant.""" - # ensure that we're a member of that tenant - tenant_id = user.get('tenantId') - self.identity_api.add_user_to_tenant(context, tenant_id, user_id) - return self.update_user(context, user_id, user) - - -class RoleController(Application): - def __init__(self): - self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() - self.token_api = token.Manager() - self.policy_api = policy.Manager() - super(RoleController, self).__init__() - - def get_user_roles(self, context, user_id, tenant_id=None): - raise NotImplemented() - - # CRUD extension - def get_role(self, context, role_id): - self.assert_admin(context) - role_ref = self.identity_api.get_role(context, role_id) - if not role_ref: - raise webob.exc.HTTPNotFound() - return {'role': role_ref} - - def create_role(self, context, role): - role = self._normalize_dict(role) - self.assert_admin(context) - role_id = uuid.uuid4().hex - role['id'] = role_id - role_ref = self.identity_api.create_role(context, role_id, role) - return {'role': role_ref} - - def delete_role(self, context, role_id): - self.assert_admin(context) - role_ref = self.identity_api.delete_role(context, role_id) - - def get_roles(self, context): - self.assert_admin(context) - roles = self.identity_api.list_roles(context) - # TODO(termie): probably inefficient at some point - return {'roles': roles} - - # COMPAT(diablo): CRUD extension - def get_role_refs(self, context, user_id): - """Ultimate hack to get around having to make role_refs first-class. - - This will basically iterate over the various roles the user has in - all tenants the user is a member of and create fake role_refs where - the id encodes the user-tenant-role information so we can look - up the appropriate data when we need to delete them. - - """ - self.assert_admin(context) - user_ref = self.identity_api.get_user(context, user_id) - tenant_ids = self.identity_api.get_tenants_for_user(context, user_id) - o = [] - for tenant_id in tenant_ids: - role_ids = self.identity_api.get_roles_for_user_and_tenant( - context, user_id, tenant_id) - for role_id in role_ids: - ref = {'roleId': role_id, - 'tenantId': tenant_id, - 'userId': user_id} - ref['id'] = urllib.urlencode(ref) - o.append(ref) - return {'roles': o} - - def create_role_ref(self, context, user_id, role): - """This is actually used for adding a user to a tenant. - - In the legacy data model adding a user to a tenant required setting - a role. - - """ - self.assert_admin(context) - # TODO(termie): for now we're ignoring the actual role - tenant_id = role.get('tenantId') - role_id = role.get('roleId') - self.identity_api.add_user_to_tenant(context, tenant_id, user_id) - self.identity_api.add_role_to_user_and_tenant( - context, user_id, tenant_id, role_id) - role_ref = self.identity_api.get_role(context, role_id) - return {'role': role_ref} - - def delete_role_ref(self, context, user_id, role_ref_id): - """This is actually used for deleting a user from a tenant. - - In the legacy data model removing a user from a tenant required - deleting a role. - - To emulate this, we encode the tenant and role in the role_ref_id, - and if this happens to be the last role for the user-tenant pair, - we remove the user from the tenant. - - """ - self.assert_admin(context) - # TODO(termie): for now we're ignoring the actual role - role_ref_ref = urlparse.parse_qs(role_ref_id) - tenant_id = role_ref_ref.get('tenantId')[0] - role_id = role_ref_ref.get('roleId')[0] - self.identity_api.remove_role_from_user_and_tenant( - context, user_id, tenant_id, role_id) - roles = self.identity_api.get_roles_for_user_and_tenant( - context, user_id, tenant_id) - if not roles: - self.identity_api.remove_user_from_tenant( - context, tenant_id, user_id) - - -class ServiceController(Application): - def __init__(self): - self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() - self.token_api = token.Manager() - self.policy_api = policy.Manager() - super(ServiceController, self).__init__() - - # CRUD extensions - # NOTE(termie): this OS-KSADM stuff is not very consistent - def get_services(self, context): - service_list = self.catalog_api.list_services(context) - service_refs = [self.catalog_api.get_service(context, x) - for x in service_list] - return {'OS-KSADM:services': service_refs} - - def get_service(self, context, service_id): - service_ref = self.catalog_api.get_service(context, service_id) - if not service_ref: - raise webob.exc.HTTPNotFound() - return {'OS-KSADM:service': service_ref} - - def delete_service(self, context, service_id): - service_ref = self.catalog_api.delete_service(context, service_id) - - def create_service(self, context, OS_KSADM_service): - service_id = uuid.uuid4().hex - service_ref = OS_KSADM_service.copy() - service_ref['id'] = service_id - new_service_ref = self.catalog_api.create_service( - context, service_id, service_ref) - return {'OS-KSADM:service': new_service_ref} - - -class VersionController(Application): +class VersionController(wsgi.Application): def __init__(self): super(VersionController, self).__init__() @@ -985,7 +336,7 @@ class VersionController(Application): raise NotImplemented() -class ExtensionsController(Application): +class ExtensionsController(wsgi.Application): def __init__(self): super(ExtensionsController, self).__init__() diff --git a/keystone/token/__init__.py b/keystone/token/__init__.py index e69de29bb2..bf971a5aba 100644 --- a/keystone/token/__init__.py +++ b/keystone/token/__init__.py @@ -0,0 +1 @@ +from keystone.token.core import * diff --git a/keystone/token/backends/__init__.py b/keystone/token/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py new file mode 100644 index 0000000000..574b2fc7c9 --- /dev/null +++ b/keystone/token/backends/kvs.py @@ -0,0 +1,15 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone.common import kvs + +class KvsToken(kvs.Base): + # Public interface + def get_token(self, token_id): + return self.db.get('token-%s' % token_id) + + def create_token(self, token_id, data): + self.db.set('token-%s' % token_id, data) + return data + + def delete_token(self, token_id): + return self.db.delete('token-%s' % token_id) From ff6af1f808bc962c816528174096b5e386c1cb9a Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 21:09:04 -0800 Subject: [PATCH 203/334] split up sql backends too --- keystone/common/sql/core.py | 415 +----------------- .../versions/001_add_initial_tables.py | 8 +- keystone/contrib/ec2/backends/sql.py | 57 +++ keystone/identity/backends/sql.py | 340 ++++++++++++++ 4 files changed, 410 insertions(+), 410 deletions(-) create mode 100644 keystone/contrib/ec2/backends/sql.py create mode 100644 keystone/identity/backends/sql.py diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index 0f870ca062..bac1ac1212 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -14,13 +14,17 @@ import sqlalchemy.pool import sqlalchemy.engine.url from keystone import config -from keystone.backends.sql import migration CONF = config.CONF -Base = declarative.declarative_base() +ModelBase = declarative.declarative_base() + + +# For exporting to other modules +Column = sql.Column +String = sql.String # Special Fields @@ -74,111 +78,8 @@ class DictBase(object): #return local.iteritems() -# Tables -class User(Base, DictBase): - __tablename__ = 'user' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - #password = sql.Column(sql.String(64)) - extra = sql.Column(JsonBlob()) - - @classmethod - def from_dict(cls, user_dict): - # shove any non-indexed properties into extra - extra = {} - for k, v in user_dict.copy().iteritems(): - # TODO(termie): infer this somehow - if k not in ['id', 'name']: - extra[k] = user_dict.pop(k) - - user_dict['extra'] = extra - return cls(**user_dict) - - def to_dict(self): - extra_copy = self.extra.copy() - extra_copy['id'] = self.id - extra_copy['name'] = self.name - return extra_copy - - -class Tenant(Base, DictBase): - __tablename__ = 'tenant' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - extra = sql.Column(JsonBlob()) - - @classmethod - def from_dict(cls, tenant_dict): - # shove any non-indexed properties into extra - extra = {} - for k, v in tenant_dict.copy().iteritems(): - # TODO(termie): infer this somehow - if k not in ['id', 'name']: - extra[k] = tenant_dict.pop(k) - - tenant_dict['extra'] = extra - return cls(**tenant_dict) - - def to_dict(self): - extra_copy = self.extra.copy() - extra_copy['id'] = self.id - extra_copy['name'] = self.name - return extra_copy - - -class Role(Base, DictBase): - __tablename__ = 'role' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64)) - - -class Metadata(Base, DictBase): - __tablename__ = 'metadata' - #__table_args__ = ( - # sql.Index('idx_metadata_usertenant', 'user', 'tenant'), - # ) - - user_id = sql.Column(sql.String(64), primary_key=True) - tenant_id = sql.Column(sql.String(64), primary_key=True) - data = sql.Column(JsonBlob()) - - -class Token(Base, DictBase): - __tablename__ = 'token' - id = sql.Column(sql.String(64), primary_key=True) - user = sql.Column(sql.String(64)) - tenant = sql.Column(sql.String(64)) - data = sql.Column(JsonBlob()) - - -class Ec2Credential(Base, DictBase): - __tablename__ = 'ec2_credential' - access = sql.Column(sql.String(64), primary_key=True) - secret = sql.Column(sql.String(64)) - user_id = sql.Column(sql.String(64)) - tenant_id = sql.Column(sql.String(64)) - - @classmethod - def from_dict(cls, user_dict): - return cls(**user_dict) - - def to_dict(self): - return dict(self.iteritems()) - - -class UserTenantMembership(Base, DictBase): - """Tenant membership join table.""" - __tablename__ = 'user_tenant_membership' - user_id = sql.Column(sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True) - tenant_id = sql.Column(sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True) - - # Backends -class SqlBase(object): +class Base(object): _MAKER = None _ENGINE = None @@ -214,305 +115,3 @@ class SqlBase(object): return sqlalchemy.orm.sessionmaker(bind=engine, autocommit=autocommit, expire_on_commit=expire_on_commit) - - -class SqlIdentity(SqlBase): - # Internal interface to manage the database - def db_sync(self): - migration.db_sync() - - # Identity interface - def authenticate(self, user_id=None, tenant_id=None, password=None): - """Authenticate based on a user, tenant and password. - - Expects the user object to have a password field and the tenant to be - in the list of tenants on the user. - - """ - user_ref = self.get_user(user_id) - tenant_ref = None - metadata_ref = None - if not user_ref or user_ref.get('password') != password: - raise AssertionError('Invalid user / password') - - tenants = self.get_tenants_for_user(user_id) - if tenant_id and tenant_id not in tenants: - raise AssertionError('Invalid tenant') - - tenant_ref = self.get_tenant(tenant_id) - if tenant_ref: - metadata_ref = self.get_metadata(user_id, tenant_id) - else: - metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) - - def get_tenant(self, tenant_id): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - if not tenant_ref: - return - return tenant_ref.to_dict() - - def get_tenant_by_name(self, tenant_name): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() - if not tenant_ref: - return - return tenant_ref.to_dict() - - def get_user(self, user_id): - session = self.get_session() - user_ref = session.query(User).filter_by(id=user_id).first() - if not user_ref: - return - return user_ref.to_dict() - - def get_user_by_name(self, user_name): - session = self.get_session() - user_ref = session.query(User).filter_by(name=user_name).first() - if not user_ref: - return - return user_ref.to_dict() - - def get_metadata(self, user_id, tenant_id): - session = self.get_session() - metadata_ref = session.query(Metadata)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - return getattr(metadata_ref, 'data', None) - - def get_role(self, role_id): - session = self.get_session() - role_ref = session.query(Role).filter_by(id=role_id).first() - return role_ref - - def list_users(self): - session = self.get_session() - user_refs = session.query(User) - return [x.to_dict() for x in user_refs] - - def list_roles(self): - session = self.get_session() - role_refs = session.query(Role) - return list(role_refs) - - # These should probably be part of the high-level API - def add_user_to_tenant(self, tenant_id, user_id): - session = self.get_session() - q = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id) - rv = q.first() - if rv: - return - - with session.begin(): - session.add(UserTenantMembership(user_id=user_id, - tenant_id=tenant_id)) - session.flush() - - def remove_user_from_tenant(self, tenant_id, user_id): - session = self.get_session() - membership_ref = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - with session.begin(): - session.delete(membership_ref) - session.flush() - - def get_tenants_for_user(self, user_id): - session = self.get_session() - membership_refs = session.query(UserTenantMembership)\ - .filter_by(user_id=user_id)\ - .all() - - return [x.tenant_id for x in membership_refs] - - def get_roles_for_user_and_tenant(self, user_id, tenant_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - if not metadata_ref: - metadata_ref = {} - return metadata_ref.get('roles', []) - - def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - is_new = False - if not metadata_ref: - is_new = True - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.add(role_id) - metadata_ref['roles'] = list(roles) - if not is_new: - self.update_metadata(user_id, tenant_id, metadata_ref) - else: - self.create_metadata(user_id, tenant_id, metadata_ref) - - def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - metadata_ref = self.get_metadata(user_id, tenant_id) - is_new = False - if not metadata_ref: - is_new = True - metadata_ref = {} - roles = set(metadata_ref.get('roles', [])) - roles.remove(role_id) - metadata_ref['roles'] = list(roles) - if not is_new: - self.update_metadata(user_id, tenant_id, metadata_ref) - else: - self.create_metadata(user_id, tenant_id, metadata_ref) - - # CRUD - def create_user(self, user_id, user): - session = self.get_session() - with session.begin(): - user_ref = User.from_dict(user) - session.add(user_ref) - session.flush() - return user_ref.to_dict() - - def update_user(self, user_id, user): - session = self.get_session() - with session.begin(): - user_ref = session.query(User).filter_by(id=user_id).first() - old_user_dict = user_ref.to_dict() - for k in user: - old_user_dict[k] = user[k] - new_user = User.from_dict(old_user_dict) - - user_ref.name = new_user.name - user_ref.extra = new_user.extra - session.flush() - return user_ref - - def delete_user(self, user_id): - session = self.get_session() - user_ref = session.query(User).filter_by(id=user_id).first() - with session.begin(): - session.delete(user_ref) - session.flush() - - def create_tenant(self, tenant_id, tenant): - session = self.get_session() - with session.begin(): - tenant_ref = Tenant.from_dict(tenant) - session.add(tenant_ref) - session.flush() - return tenant_ref.to_dict() - - def update_tenant(self, tenant_id, tenant): - session = self.get_session() - with session.begin(): - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - old_tenant_dict = tenant_ref.to_dict() - for k in tenant: - old_tenant_dict[k] = tenant[k] - new_tenant = Tenant.from_dict(old_tenant_dict) - - tenant_ref.name = new_tenant.name - tenant_ref.extra = new_tenant.extra - session.flush() - return tenant_ref - - def delete_tenant(self, tenant_id): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - with session.begin(): - session.delete(tenant_ref) - session.flush() - - def create_metadata(self, user_id, tenant_id, metadata): - session = self.get_session() - with session.begin(): - session.add(Metadata(user_id=user_id, - tenant_id=tenant_id, - data=metadata)) - session.flush() - return metadata - - def update_metadata(self, user_id, tenant_id, metadata): - session = self.get_session() - with session.begin(): - metadata_ref = session.query(Metadata)\ - .filter_by(user_id=user_id)\ - .filter_by(tenant_id=tenant_id)\ - .first() - data = metadata_ref.data.copy() - for k in metadata: - data[k] = metadata[k] - metadata_ref.data = data - session.flush() - return metadata_ref - - def delete_metadata(self, user_id, tenant_id): - self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) - return None - - def create_role(self, role_id, role): - session = self.get_session() - with session.begin(): - session.add(Role(**role)) - session.flush() - return role - - def update_role(self, role_id, role): - session = self.get_session() - with session.begin(): - role_ref = session.query(Role).filter_by(id=role_id).first() - for k in role: - role_ref[k] = role[k] - session.flush() - return role_ref - - def delete_role(self, role_id): - session = self.get_session() - role_ref = session.query(Role).filter_by(id=role_id).first() - with session.begin(): - session.delete(role_ref) - - -class SqlToken(SqlBase): - pass - - -class SqlCatalog(SqlBase): - pass - - -class SqlEc2(SqlBase): - # Internal interface to manage the database - def db_sync(self): - migration.db_sync() - - def get_credential(self, credential_id): - session = self.get_session() - credential_ref = session.query(Ec2Credential)\ - .filter_by(access=credential_id).first() - if not credential_ref: - return - return credential_ref.to_dict() - - def list_credentials(self, user_id): - session = self.get_session() - credential_refs = session.query(Ec2Credential)\ - .filter_by(user_id=user_id) - return [x.to_dict() for x in credential_refs] - - # CRUD - def create_credential(self, credential_id, credential): - session = self.get_session() - with session.begin(): - credential_ref = Ec2Credential.from_dict(credential) - session.add(credential_ref) - session.flush() - return credential_ref.to_dict() - - def delete_credential(self, credential_id): - session = self.get_session() - credential_ref = session.query(Ec2Credential)\ - .filter_by(access=credential_id).first() - with session.begin(): - session.delete(credential_ref) - session.flush() diff --git a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py index 92c9e6cd1e..5875817e5b 100644 --- a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py +++ b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py @@ -1,13 +1,17 @@ from sqlalchemy import * from migrate import * -from keystone.backends import sql +from keystone.common import sql + +# these are to make sure all the models we care about are defined +import keystone.identity.backends.sql +import keystone.contrib.ec2.backends.sql def upgrade(migrate_engine): # Upgrade operations go here. Don't create your own engine; bind # migrate_engine to your metadata - sql.Base.metadata.create_all(migrate_engine) + sql.ModelBase.metadata.create_all(migrate_engine) def downgrade(migrate_engine): diff --git a/keystone/contrib/ec2/backends/sql.py b/keystone/contrib/ec2/backends/sql.py new file mode 100644 index 0000000000..3f786a6a6e --- /dev/null +++ b/keystone/contrib/ec2/backends/sql.py @@ -0,0 +1,57 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone.common import sql +from keystone.common.sql import migration + + +class Ec2Credential(sql.ModelBase, sql.DictBase): + __tablename__ = 'ec2_credential' + access = sql.Column(sql.String(64), primary_key=True) + secret = sql.Column(sql.String(64)) + user_id = sql.Column(sql.String(64)) + tenant_id = sql.Column(sql.String(64)) + + @classmethod + def from_dict(cls, user_dict): + return cls(**user_dict) + + def to_dict(self): + return dict(self.iteritems()) + + +class Ec2(sql.Base): + # Internal interface to manage the database + def db_sync(self): + migration.db_sync() + + def get_credential(self, credential_id): + session = self.get_session() + credential_ref = session.query(Ec2Credential)\ + .filter_by(access=credential_id).first() + if not credential_ref: + return + return credential_ref.to_dict() + + def list_credentials(self, user_id): + session = self.get_session() + credential_refs = session.query(Ec2Credential)\ + .filter_by(user_id=user_id) + return [x.to_dict() for x in credential_refs] + + # CRUD + def create_credential(self, credential_id, credential): + session = self.get_session() + with session.begin(): + credential_ref = Ec2Credential.from_dict(credential) + session.add(credential_ref) + session.flush() + return credential_ref.to_dict() + + def delete_credential(self, credential_id): + session = self.get_session() + credential_ref = session.query(Ec2Credential)\ + .filter_by(access=credential_id).first() + with session.begin(): + session.delete(credential_ref) + session.flush() + diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py new file mode 100644 index 0000000000..78d3008a2c --- /dev/null +++ b/keystone/identity/backends/sql.py @@ -0,0 +1,340 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone.common import sql +from keystone.common.sql import migration + +class User(sql.ModelBase, sql.DictBase): + __tablename__ = 'user' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + #password = sql.Column(sql.String(64)) + extra = sql.Column(sql.JsonBlob()) + + @classmethod + def from_dict(cls, user_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in user_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = user_dict.pop(k) + + user_dict['extra'] = extra + return cls(**user_dict) + + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy + + +class Tenant(sql.ModelBase, sql.DictBase): + __tablename__ = 'tenant' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + extra = sql.Column(JsonBlob()) + + @classmethod + def from_dict(cls, tenant_dict): + # shove any non-indexed properties into extra + extra = {} + for k, v in tenant_dict.copy().iteritems(): + # TODO(termie): infer this somehow + if k not in ['id', 'name']: + extra[k] = tenant_dict.pop(k) + + tenant_dict['extra'] = extra + return cls(**tenant_dict) + + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + extra_copy['name'] = self.name + return extra_copy + + +class Role(sql.ModelBase, sql.DictBase): + __tablename__ = 'role' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64)) + + +class Metadata(sql.ModelBase, sql.DictBase): + __tablename__ = 'metadata' + #__table_args__ = ( + # sql.Index('idx_metadata_usertenant', 'user', 'tenant'), + # ) + + user_id = sql.Column(sql.String(64), primary_key=True) + tenant_id = sql.Column(sql.String(64), primary_key=True) + data = sql.Column(JsonBlob()) + + +class UserTenantMembership(sql.ModelBase, sql.DictBase): + """Tenant membership join table.""" + __tablename__ = 'user_tenant_membership' + user_id = sql.Column(sql.String(64), + sql.ForeignKey('user.id'), + primary_key=True) + tenant_id = sql.Column(sql.String(64), + sql.ForeignKey('tenant.id'), + primary_key=True) + + +class SqlIdentity(sql.Base): + # Internal interface to manage the database + def db_sync(self): + migration.db_sync() + + # Identity interface + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. + + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. + + """ + user_ref = self.get_user(user_id) + tenant_ref = None + metadata_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') + + tenants = self.get_tenants_for_user(user_id) + if tenant_id and tenant_id not in tenants: + raise AssertionError('Invalid tenant') + + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + metadata_ref = self.get_metadata(user_id, tenant_id) + else: + metadata_ref = {} + return (user_ref, tenant_ref, metadata_ref) + + def get_tenant(self, tenant_id): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + if not tenant_ref: + return + return tenant_ref.to_dict() + + def get_tenant_by_name(self, tenant_name): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(name=tenant_name).first() + if not tenant_ref: + return + return tenant_ref.to_dict() + + def get_user(self, user_id): + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + if not user_ref: + return + return user_ref.to_dict() + + def get_user_by_name(self, user_name): + session = self.get_session() + user_ref = session.query(User).filter_by(name=user_name).first() + if not user_ref: + return + return user_ref.to_dict() + + def get_metadata(self, user_id, tenant_id): + session = self.get_session() + metadata_ref = session.query(Metadata)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + return getattr(metadata_ref, 'data', None) + + def get_role(self, role_id): + session = self.get_session() + role_ref = session.query(Role).filter_by(id=role_id).first() + return role_ref + + def list_users(self): + session = self.get_session() + user_refs = session.query(User) + return [x.to_dict() for x in user_refs] + + def list_roles(self): + session = self.get_session() + role_refs = session.query(Role) + return list(role_refs) + + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + session = self.get_session() + q = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id) + rv = q.first() + if rv: + return + + with session.begin(): + session.add(UserTenantMembership(user_id=user_id, + tenant_id=tenant_id)) + session.flush() + + def remove_user_from_tenant(self, tenant_id, user_id): + session = self.get_session() + membership_ref = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + with session.begin(): + session.delete(membership_ref) + session.flush() + + def get_tenants_for_user(self, user_id): + session = self.get_session() + membership_refs = session.query(UserTenantMembership)\ + .filter_by(user_id=user_id)\ + .all() + + return [x.tenant_id for x in membership_refs] + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + if not metadata_ref: + metadata_ref = {} + return metadata_ref.get('roles', []) + + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + is_new = False + if not metadata_ref: + is_new = True + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.add(role_id) + metadata_ref['roles'] = list(roles) + if not is_new: + self.update_metadata(user_id, tenant_id, metadata_ref) + else: + self.create_metadata(user_id, tenant_id, metadata_ref) + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + metadata_ref = self.get_metadata(user_id, tenant_id) + is_new = False + if not metadata_ref: + is_new = True + metadata_ref = {} + roles = set(metadata_ref.get('roles', [])) + roles.remove(role_id) + metadata_ref['roles'] = list(roles) + if not is_new: + self.update_metadata(user_id, tenant_id, metadata_ref) + else: + self.create_metadata(user_id, tenant_id, metadata_ref) + + # CRUD + def create_user(self, user_id, user): + session = self.get_session() + with session.begin(): + user_ref = User.from_dict(user) + session.add(user_ref) + session.flush() + return user_ref.to_dict() + + def update_user(self, user_id, user): + session = self.get_session() + with session.begin(): + user_ref = session.query(User).filter_by(id=user_id).first() + old_user_dict = user_ref.to_dict() + for k in user: + old_user_dict[k] = user[k] + new_user = User.from_dict(old_user_dict) + + user_ref.name = new_user.name + user_ref.extra = new_user.extra + session.flush() + return user_ref + + def delete_user(self, user_id): + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + with session.begin(): + session.delete(user_ref) + session.flush() + + def create_tenant(self, tenant_id, tenant): + session = self.get_session() + with session.begin(): + tenant_ref = Tenant.from_dict(tenant) + session.add(tenant_ref) + session.flush() + return tenant_ref.to_dict() + + def update_tenant(self, tenant_id, tenant): + session = self.get_session() + with session.begin(): + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + old_tenant_dict = tenant_ref.to_dict() + for k in tenant: + old_tenant_dict[k] = tenant[k] + new_tenant = Tenant.from_dict(old_tenant_dict) + + tenant_ref.name = new_tenant.name + tenant_ref.extra = new_tenant.extra + session.flush() + return tenant_ref + + def delete_tenant(self, tenant_id): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + with session.begin(): + session.delete(tenant_ref) + session.flush() + + def create_metadata(self, user_id, tenant_id, metadata): + session = self.get_session() + with session.begin(): + session.add(Metadata(user_id=user_id, + tenant_id=tenant_id, + data=metadata)) + session.flush() + return metadata + + def update_metadata(self, user_id, tenant_id, metadata): + session = self.get_session() + with session.begin(): + metadata_ref = session.query(Metadata)\ + .filter_by(user_id=user_id)\ + .filter_by(tenant_id=tenant_id)\ + .first() + data = metadata_ref.data.copy() + for k in metadata: + data[k] = metadata[k] + metadata_ref.data = data + session.flush() + return metadata_ref + + def delete_metadata(self, user_id, tenant_id): + self.db.delete('metadata-%s-%s' % (tenant_id, user_id)) + return None + + def create_role(self, role_id, role): + session = self.get_session() + with session.begin(): + session.add(Role(**role)) + session.flush() + return role + + def update_role(self, role_id, role): + session = self.get_session() + with session.begin(): + role_ref = session.query(Role).filter_by(id=role_id).first() + for k in role: + role_ref[k] = role[k] + session.flush() + return role_ref + + def delete_role(self, role_id): + session = self.get_session() + role_ref = session.query(Role).filter_by(id=role_id).first() + with session.begin(): + session.delete(role_ref) + From e2f04f224e36c16430d7f7430b05d625ca5145a1 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 21:10:08 -0800 Subject: [PATCH 204/334] fix some imports --- keystone/catalog/core.py | 6 ------ keystone/common/sql/__init__.py | 2 +- keystone/common/sql/util.py | 2 +- keystone/common/wsgi.py | 11 ++++++++--- keystone/config.py | 2 +- keystone/contrib/ec2/backends/sql.py | 4 ---- keystone/contrib/ec2/core.py | 2 +- keystone/identity/core.py | 18 +++++++++++------- keystone/middleware/internal.py | 2 +- keystone/policy/__init__.py | 1 + keystone/policy/core.py | 2 +- keystone/test.py | 6 +++--- keystone/token/core.py | 2 +- tests/test_keystoneclient_sql.py | 4 ++-- 14 files changed, 32 insertions(+), 32 deletions(-) diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index 8177f2cffe..31e416885c 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -3,9 +3,6 @@ """Main entry point into the Catalog service.""" from keystone import config -from keystone import identity -from keystone import token -from keystone import policy from keystone.common import manager from keystone.common import wsgi @@ -31,9 +28,6 @@ class Manager(manager.Manager): class ServiceController(wsgi.Application): def __init__(self): self.catalog_api = Manager() - self.identity_api = identity.Manager() - self.token_api = token.Manager() - self.policy_api = policy.Manager() super(ServiceController, self).__init__() # CRUD extensions diff --git a/keystone/common/sql/__init__.py b/keystone/common/sql/__init__.py index c52d6cbce2..ae31c7029e 100644 --- a/keystone/common/sql/__init__.py +++ b/keystone/common/sql/__init__.py @@ -1 +1 @@ -from keystone.backends.sql.core import * +from keystone.common.sql.core import * diff --git a/keystone/common/sql/util.py b/keystone/common/sql/util.py index e96ac6c631..a181c28498 100644 --- a/keystone/common/sql/util.py +++ b/keystone/common/sql/util.py @@ -3,7 +3,7 @@ import os from keystone import config -from keystone.backends.sql import migration +from keystone.common.sql import migration CONF = config.CONF diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index b7b244fa6f..fafa969435 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -19,6 +19,7 @@ """Utility methods for working with WSGI servers.""" +import json import logging import sys @@ -31,6 +32,8 @@ import webob import webob.dec import webob.exc +from keystone.common import utils + class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -146,7 +149,7 @@ class BaseApplication(object): raise NotImplementedError('You must implement __call__') -class Application(wsgi.BaseApplication): +class Application(BaseApplication): @webob.dec.wsgify def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] @@ -375,7 +378,7 @@ class ComposingRouter(Router): super(ComposingRouter, self).__init__(mapper) -class ComposableRouter(object): +class ComposableRouter(Router): """Router that supports use by ComposingRouter.""" def __init__(self, mapper=None): @@ -394,7 +397,9 @@ class ExtensionRouter(Router): Expects to be subclassed. """ - def __init__(self, application, mapper): + def __init__(self, application, mapper=None): + if mapper is None: + mapper = routes.Mapper() self.application = application self.add_routes(mapper) mapper.connect('{path_info:.*}', controller=self.application) diff --git a/keystone/config.py b/keystone/config.py index f3064dc7be..a89c6e9031 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -5,7 +5,7 @@ import logging import sys import os -from keystone import cfg +from keystone.common import cfg gettext.install('keystone', unicode=1) diff --git a/keystone/contrib/ec2/backends/sql.py b/keystone/contrib/ec2/backends/sql.py index 3f786a6a6e..a0464ba855 100644 --- a/keystone/contrib/ec2/backends/sql.py +++ b/keystone/contrib/ec2/backends/sql.py @@ -20,10 +20,6 @@ class Ec2Credential(sql.ModelBase, sql.DictBase): class Ec2(sql.Base): - # Internal interface to manage the database - def db_sync(self): - migration.db_sync() - def get_credential(self, credential_id): session = self.get_session() credential_ref = session.query(Ec2Credential)\ diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 1c83689c00..e12c9efce8 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -30,7 +30,7 @@ class Manager(manager.Manager): class Ec2Extension(wsgi.ExtensionRouter): - def add_routes(self, mapper) + def add_routes(self, mapper): ec2_controller = Ec2Controller() # validation mapper.connect('/ec2tokens', diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 4400d7f6eb..3d46b66d41 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -2,8 +2,12 @@ """Main entry point into the Identity service.""" +from keystone import catalog from keystone import config -from keystone import manager +from keystone import policy +from keystone import token +from keystone.common import manager +from keystone.common import wsgi CONF = config.CONF @@ -65,9 +69,9 @@ class AdminRouter(wsgi.ComposableRouter): conditions=dict(method=['GET'])) -class TenantController(Application): +class TenantController(wsgi.Application): def __init__(self): - self.identity_api = identity.Manager() + self.identity_api = Manager() self.policy_api = policy.Manager() self.token_api = token.Manager() super(TenantController, self).__init__() @@ -148,10 +152,10 @@ class TenantController(Application): return o -class UserController(Application): +class UserController(wsgi.Application): def __init__(self): self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() + self.identity_api = Manager() self.policy_api = policy.Manager() self.token_api = token.Manager() super(UserController, self).__init__() @@ -211,10 +215,10 @@ class UserController(Application): return self.update_user(context, user_id, user) -class RoleController(Application): +class RoleController(wsgi.Application): def __init__(self): self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() + self.identity_api = Manager() self.token_api = token.Manager() self.policy_api = policy.Manager() super(RoleController, self).__init__() diff --git a/keystone/middleware/internal.py b/keystone/middleware/internal.py index ccf68b82d1..389c9bf0f8 100644 --- a/keystone/middleware/internal.py +++ b/keystone/middleware/internal.py @@ -3,7 +3,7 @@ import json from keystone import config -from keystone import wsgi +from keystone.common import wsgi CONF = config.CONF diff --git a/keystone/policy/__init__.py b/keystone/policy/__init__.py index e69de29bb2..d16de59fc2 100644 --- a/keystone/policy/__init__.py +++ b/keystone/policy/__init__.py @@ -0,0 +1 @@ +from keystone.policy.core import * diff --git a/keystone/policy/core.py b/keystone/policy/core.py index 4aa24df759..87ad743cd6 100644 --- a/keystone/policy/core.py +++ b/keystone/policy/core.py @@ -3,7 +3,7 @@ """Main entry point into the Policy service.""" from keystone import config -from keystone import manager +from keystone.common import manager CONF = config.CONF diff --git a/keystone/test.py b/keystone/test.py index b3473b5486..04502f80b3 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -11,10 +11,10 @@ from paste import deploy from keystone import catalog from keystone import config from keystone import identity -from keystone import logging from keystone import token -from keystone import utils -from keystone import wsgi +from keystone.common import logging +from keystone.common import utils +from keystone.common import wsgi ROOTDIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/keystone/token/core.py b/keystone/token/core.py index d3e9a5f09a..c48ca0be05 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -3,7 +3,7 @@ """Main entry point into the Token service.""" from keystone import config -from keystone import manager +from keystone.common import manager CONF = config.CONF diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py index 2adcc745d9..ce8ce01151 100644 --- a/tests/test_keystoneclient_sql.py +++ b/tests/test_keystoneclient_sql.py @@ -1,8 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 from keystone import config from keystone import test -from keystone.backends.sql import util as sql_util -from keystone.backends.sql import migration +from keystone.common.sql import util as sql_util +from keystone.common.sql import migration import test_keystoneclient From fc79bbe7f4f6622a570d704e3e0d4ac248a30ec4 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 21:15:14 -0800 Subject: [PATCH 205/334] update some names --- bin/keystone | 2 +- bin/keystone-manage | 2 +- etc/keystone.conf | 16 ++++++++-------- keystone/catalog/backends/kvs.py | 2 +- keystone/identity/backends/sql.py | 2 +- keystone/middleware/__init__.py | 2 +- keystone/middleware/{internal.py => core.py} | 0 keystone/token/backends/kvs.py | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) rename keystone/middleware/{internal.py => core.py} (100%) diff --git a/bin/keystone b/bin/keystone index 6e0f43982a..5f75279baf 100755 --- a/bin/keystone +++ b/bin/keystone @@ -19,7 +19,7 @@ if os.path.exists(os.path.join(possible_topdir, from paste import deploy from keystone import config -from keystone import wsgi +from keystone.common import wsgi CONF = config.CONF diff --git a/bin/keystone-manage b/bin/keystone-manage index 63c1aac12f..d290c8cdfb 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -19,7 +19,7 @@ if os.path.exists(os.path.join(possible_topdir, from keystone import config -from keystone import utils +from keystone.common import utils CONF = config.CONF diff --git a/etc/keystone.conf b/etc/keystone.conf index f4c38e072c..bbc84b78a2 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -23,23 +23,23 @@ max_pool_size = 10 pool_timeout = 200 [identity] -driver = keystone.backends.kvs.KvsIdentity +driver = keystone.identity.backends.kvs.Identity [catalog] -driver = keystone.backends.templated.TemplatedCatalog +driver = keystone.catalog.backends.templated.TemplatedCatalog template_file = ./etc/default_catalog.templates [token] -driver = keystone.backends.kvs.KvsToken +driver = keystone.token.backends.kvs.Token [policy] -driver = keystone.backends.policy.SimpleMatch +driver = keystone.policy.backends.simple.SimpleMatch [ec2] -driver = keystone.backends.kvs.KvsEc2 +driver = keystone.contrib.ec2.backends.kvs.Ec2 [filter:debug] -paste.filter_factory = keystone.wsgi:Debug.factory +paste.filter_factory = keystone.common.wsgi:Debug.factory [filter:token_auth] paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory @@ -51,10 +51,10 @@ paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory [filter:crud_extension] -paste.filter_factory = keystone.service:AdminCrudExtension.factory +paste.filter_factory = keystone.contrib.admin_crud:CrudExtension.factory [filter:ec2_extension] -paste.filter_factory = keystone.service:Ec2Extension.factory +paste.filter_factory = keystone.contrib.ec2:Ec2Extension.factory [app:public_service] paste.app_factory = keystone.service:public_app_factory diff --git a/keystone/catalog/backends/kvs.py b/keystone/catalog/backends/kvs.py index 84d1857254..69f45492f1 100644 --- a/keystone/catalog/backends/kvs.py +++ b/keystone/catalog/backends/kvs.py @@ -4,7 +4,7 @@ from keystone.common import kvs -class KvsCatalog(kvs.Base): +class Catalog(kvs.Base): # Public interface def get_catalog(self, user_id, tenant_id, metadata=None): return self.db.get('catalog-%s-%s' % (tenant_id, user_id)) diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index 78d3008a2c..f12e852b23 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -82,7 +82,7 @@ class UserTenantMembership(sql.ModelBase, sql.DictBase): primary_key=True) -class SqlIdentity(sql.Base): +class Identity(sql.Base): # Internal interface to manage the database def db_sync(self): migration.db_sync() diff --git a/keystone/middleware/__init__.py b/keystone/middleware/__init__.py index 1593e6e2f4..e2e9a993df 100644 --- a/keystone/middleware/__init__.py +++ b/keystone/middleware/__init__.py @@ -1 +1 @@ -from keystone.middleware.internal import * +from keystone.middleware.core import * diff --git a/keystone/middleware/internal.py b/keystone/middleware/core.py similarity index 100% rename from keystone/middleware/internal.py rename to keystone/middleware/core.py diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py index 574b2fc7c9..8686a3dc5a 100644 --- a/keystone/token/backends/kvs.py +++ b/keystone/token/backends/kvs.py @@ -2,7 +2,7 @@ from keystone.common import kvs -class KvsToken(kvs.Base): +class Token(kvs.Base): # Public interface def get_token(self, token_id): return self.db.get('token-%s' % token_id) From f22623491758097445d9dfcceb3a307eea5979bf Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 21:21:29 -0800 Subject: [PATCH 206/334] update tests --- tests/backend_sql.conf | 2 +- tests/keystone_compat_diablo.conf | 52 ------------------------------- tests/test_backend_kvs.py | 10 +++--- tests/test_backend_sql.py | 6 ++-- tests/test_legacy_compat.py | 4 +-- tests/test_novaclient_compat.py | 4 +-- 6 files changed, 14 insertions(+), 64 deletions(-) delete mode 100644 tests/keystone_compat_diablo.conf diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf index ca527e9af2..30e8f6f597 100644 --- a/tests/backend_sql.conf +++ b/tests/backend_sql.conf @@ -6,4 +6,4 @@ max_pool_size = 10 pool_timeout = 200 [identity] -driver = keystone.backends.sql.SqlIdentity +driver = keystone.identity.backends.sql.Identity diff --git a/tests/keystone_compat_diablo.conf b/tests/keystone_compat_diablo.conf deleted file mode 100644 index 5db43f6f7c..0000000000 --- a/tests/keystone_compat_diablo.conf +++ /dev/null @@ -1,52 +0,0 @@ -[DEFAULT] -public_port = 5000 -admin_port = 35357 -admin_token = ADMIN - -[identity] -driver = keystone.backends.kvs.KvsIdentity - -[catalog] -driver = keystone.backends.kvs.KvsCatalog - -[token] -driver = keystone.backends.kvs.KvsToken - -[policy] -driver = keystone.backends.policy.SimpleMatch - - -[filter:debug] -paste.filter_factory = keystone.wsgi:Debug.factory - -[filter:token_auth] -paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory - -[filter:admin_token_auth] -paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory - -[filter:json_body] -paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory - - -[app:keystone_service] -paste.app_factory = keystone.service:service_app_factory - -[app:keystone_admin] -paste.app_factory = keystone.service:admin_app_factory - - -[pipeline:keystone_service_api] -pipeline = token_auth admin_token_auth json_body debug keystone_service - -[pipeline:keystone_admin_api] -pipeline = token_auth admin_token_auth json_body debug keystone_admin - - -[composite:main] -use = egg:Paste#urlmap -/v2.0 = keystone_service_api - -[composite:admin] -use = egg:Paste#urlmap -/v2.0 = keystone_admin_api diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index 85b9fef6c7..fe68dd78e4 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -1,7 +1,9 @@ import uuid from keystone import test -from keystone.backends import kvs +from keystone.identity.backends import kvs as identity_kvs +from keystone.token.backends import kvs as token_kvs +from keystone.catalog.backends import kvs as catalog_kvs import test_backend import default_fixtures @@ -10,14 +12,14 @@ import default_fixtures class KvsIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(KvsIdentity, self).setUp() - self.identity_api = kvs.KvsIdentity(db={}) + self.identity_api = identity_kvs.Identity(db={}) self.load_fixtures(default_fixtures) class KvsToken(test.TestCase): def setUp(self): super(KvsToken, self).setUp() - self.token_api = kvs.KvsToken(db={}) + self.token_api = token_kvs.Token(db={}) def test_token_crud(self): token_id = uuid.uuid4().hex @@ -37,7 +39,7 @@ class KvsToken(test.TestCase): class KvsCatalog(test.TestCase): def setUp(self): super(KvsCatalog, self).setUp() - self.catalog_api = kvs.KvsCatalog(db={}) + self.catalog_api = catalog_kvs.Catalog(db={}) self._load_fixtures() def _load_fixtures(self): diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index faaa3b21df..fe137f525c 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -3,8 +3,8 @@ import uuid from keystone import config from keystone import test -from keystone.backends import sql -from keystone.backends.sql import util as sql_util +from keystone.common.sql import util as sql_util +from keystone.identity.backends import sql as identity_sql import test_backend import default_fixtures @@ -25,7 +25,7 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests): test.testsdir('test_overrides.conf'), test.testsdir('backend_sql.conf')]) sql_util.setup_test_database() - self.identity_api = sql.SqlIdentity() + self.identity_api = identity_sql.Identity() self.load_fixtures(default_fixtures) diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index fca00b6848..ff6b7fdbc2 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -6,9 +6,9 @@ import sys from nose import exc from keystone import config -from keystone import logging from keystone import test -from keystone import utils +from keystone.common import logging +from keystone.common import utils CONF = config.CONF diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index 203fa9399e..b4f6dc7928 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -4,9 +4,9 @@ import os import sys from keystone import config -from keystone import logging from keystone import test -from keystone import utils +from keystone.common import logging +from keystone.common import utils import default_fixtures From 89c378c2400d697059b3e5d81f65814424604c05 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 18 Jan 2012 21:46:39 -0800 Subject: [PATCH 207/334] fix pep8 --- keystone/catalog/backends/kvs.py | 3 --- keystone/catalog/core.py | 5 ++++- keystone/common/sql/core.py | 1 + keystone/contrib/ec2/backends/kvs.py | 1 - keystone/contrib/ec2/backends/sql.py | 1 - keystone/contrib/ec2/core.py | 4 ++-- keystone/identity/backends/kvs.py | 1 - keystone/identity/backends/sql.py | 6 +++--- keystone/identity/core.py | 8 ++++++-- keystone/service.py | 3 +-- keystone/token/backends/kvs.py | 1 + 11 files changed, 18 insertions(+), 16 deletions(-) diff --git a/keystone/catalog/backends/kvs.py b/keystone/catalog/backends/kvs.py index 69f45492f1..8ee781ba94 100644 --- a/keystone/catalog/backends/kvs.py +++ b/keystone/catalog/backends/kvs.py @@ -37,6 +37,3 @@ class Catalog(kvs.Base): def _create_catalog(self, user_id, tenant_id, data): self.db.set('catalog-%s-%s' % (tenant_id, user_id), data) return data - - - diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index 31e416885c..a99c20571e 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -2,6 +2,10 @@ """Main entry point into the Catalog service.""" +import uuid + +import webob.exc + from keystone import config from keystone.common import manager from keystone.common import wsgi @@ -54,4 +58,3 @@ class ServiceController(wsgi.Application): new_service_ref = self.catalog_api.create_service( context, service_id, service_ref) return {'OS-KSADM:service': new_service_ref} - diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index bac1ac1212..9c51d782bf 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -25,6 +25,7 @@ ModelBase = declarative.declarative_base() # For exporting to other modules Column = sql.Column String = sql.String +ForeignKey = sql.ForeignKey # Special Fields diff --git a/keystone/contrib/ec2/backends/kvs.py b/keystone/contrib/ec2/backends/kvs.py index 47975ed5cb..6aa2fe81d7 100644 --- a/keystone/contrib/ec2/backends/kvs.py +++ b/keystone/contrib/ec2/backends/kvs.py @@ -29,4 +29,3 @@ class Ec2(kvs.Base): credential_list.remove(credential_id) self.db.set('credential_list', list(credential_list)) return None - diff --git a/keystone/contrib/ec2/backends/sql.py b/keystone/contrib/ec2/backends/sql.py index a0464ba855..3105660e43 100644 --- a/keystone/contrib/ec2/backends/sql.py +++ b/keystone/contrib/ec2/backends/sql.py @@ -50,4 +50,3 @@ class Ec2(sql.Base): with session.begin(): session.delete(credential_ref) session.flush() - diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index e12c9efce8..1332b6431c 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -2,6 +2,8 @@ """Main entry point into the EC2 Credentials service.""" +import uuid + from keystone import catalog from keystone import config from keystone import identity @@ -153,5 +155,3 @@ class Ec2Controller(wsgi.Application): # TODO(termie): validate that this request is valid for given user # tenant return self.ec2_api.delete_credential(context, credential_id) - - diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index 233884c62e..ffdfb4f84c 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -175,4 +175,3 @@ class Identity(kvs.Base): role_list.remove(role_id) self.db.set('role_list', list(role_list)) return None - diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index f12e852b23..634d895fe2 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -3,6 +3,7 @@ from keystone.common import sql from keystone.common.sql import migration + class User(sql.ModelBase, sql.DictBase): __tablename__ = 'user' id = sql.Column(sql.String(64), primary_key=True) @@ -33,7 +34,7 @@ class Tenant(sql.ModelBase, sql.DictBase): __tablename__ = 'tenant' id = sql.Column(sql.String(64), primary_key=True) name = sql.Column(sql.String(64), unique=True) - extra = sql.Column(JsonBlob()) + extra = sql.Column(sql.JsonBlob()) @classmethod def from_dict(cls, tenant_dict): @@ -68,7 +69,7 @@ class Metadata(sql.ModelBase, sql.DictBase): user_id = sql.Column(sql.String(64), primary_key=True) tenant_id = sql.Column(sql.String(64), primary_key=True) - data = sql.Column(JsonBlob()) + data = sql.Column(sql.JsonBlob()) class UserTenantMembership(sql.ModelBase, sql.DictBase): @@ -337,4 +338,3 @@ class Identity(sql.Base): role_ref = session.query(Role).filter_by(id=role_id).first() with session.begin(): session.delete(role_ref) - diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 3d46b66d41..1440951342 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -2,6 +2,12 @@ """Main entry point into the Identity service.""" +import uuid +import urllib +import urlparse + +import webob.exc + from keystone import catalog from keystone import config from keystone import policy @@ -317,5 +323,3 @@ class RoleController(wsgi.Application): if not roles: self.identity_api.remove_user_from_tenant( context, tenant_id, user_id) - - diff --git a/keystone/service.py b/keystone/service.py index bdb5318199..186f1cf763 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -37,7 +37,6 @@ class AdminRouter(wsgi.ComposingRouter): action='endpoints', conditions=dict(method=['GET'])) - # Miscellaneous Operations version_controller = VersionController() mapper.connect('/', @@ -52,7 +51,7 @@ class AdminRouter(wsgi.ComposingRouter): conditions=dict(method=['GET'])) identity_router = identity.AdminRouter() routers = [identity_router] - super(AdminRouter, self).__init__(mapper, identity_router) + super(AdminRouter, self).__init__(mapper, routers) class PublicRouter(wsgi.ComposingRouter): diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py index 8686a3dc5a..b7c25fb7bf 100644 --- a/keystone/token/backends/kvs.py +++ b/keystone/token/backends/kvs.py @@ -2,6 +2,7 @@ from keystone.common import kvs + class Token(kvs.Base): # Public interface def get_token(self, token_id): From 781feaf6a8efb015db1be732f025c2d4339ab656 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 19 Jan 2012 12:41:08 -0800 Subject: [PATCH 208/334] add some docs that got overwritten last night --- keystone/identity/backends/kvs.py | 3 +- keystone/identity/backends/sql.py | 3 +- keystone/identity/core.py | 143 ++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index ffdfb4f84c..19b5d220c6 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -1,9 +1,10 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +from keystone import identity from keystone.common import kvs -class Identity(kvs.Base): +class Identity(kvs.Base, identity.Driver): # Public interface def authenticate(self, user_id=None, tenant_id=None, password=None): """Authenticate based on a user, tenant and password. diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index 634d895fe2..b2a3ffad34 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -1,5 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +from keystone import identity from keystone.common import sql from keystone.common.sql import migration @@ -83,7 +84,7 @@ class UserTenantMembership(sql.ModelBase, sql.DictBase): primary_key=True) -class Identity(sql.Base): +class Identity(sql.Base, identity.Driver): # Internal interface to manage the database def db_sync(self): migration.db_sync() diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 1440951342..9452b909ad 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -34,6 +34,149 @@ class Manager(manager.Manager): super(Manager, self).__init__(CONF.identity.driver) +class Driver(object): + """Interface description for an Identity driver.""" + + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate a given user, tenant and password. + + Returns: (user, tenant, metadata). + + """ + raise NotImplementedError() + + def get_tenant(self, tenant_id): + """Get a tenant by id. + + Returns: tenant_ref or None. + + """ + raise NotImplementedError() + + def get_tenant_by_name(self, tenant_name): + """Get a tenant by name. + + Returns: tenant_ref or None. + + """ + raise NotImplementedError() + + def get_user(self, user_id): + """Get a user by id. + + Returns: user_ref or None. + + """ + raise NotImplementedError() + + def get_user_by_name(self, user_name): + """Get a user by name. + + Returns: user_ref or None. + + """ + raise NotImplementedError() + + def get_role(self, role_id): + """Get a role by id. + + Returns: role_ref or None. + + """ + raise NotImplementedError() + + def list_users(self): + """List all users in the system. + + NOTE(termie): I'd prefer if this listed only the users for a given + tenant. + + Returns: a list of user_refs or an empty list. + + """ + raise NotImplementedError() + + def list_roles(self): + """List all roles in the system. + + Returns: a list of role_refs or an empty list. + + """ + raise NotImplementedError() + + # NOTE(termie): six calls below should probably be exposed by the api + # more clearly when the api redesign happens + def add_user_to_tenant(self, tenant_id, user_id): + raise NotImplementedError() + + def remove_user_from_tenant(self, tenant_id, user_id): + raise NotImplementedError() + + def get_tenants_for_user(self, user_id): + """Get the tenants associated with a given user. + + Returns: a list of tenant ids. + + """ + raise NotImplementedError() + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + """Get the roles associated with a user within given tenant. + + Returns: a list of role ids. + + """ + raise NotImplementedError() + + def add_role_for_user_and_tenant(self, user_id, tenant_id, role_id): + """Add a role to a user within given tenant.""" + raise NotImplementedError() + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + """Remove a role from a user within given tenant.""" + raise NotImplementedError() + + # user crud + def create_user(self, user_id, user): + raise NotImplementedError() + + def update_user(self, user_id, user): + raise NotImplementedError() + + def delete_user(self, user_id): + raise NotImplementedError() + + # tenant crud + def create_tenant(self, tenant_id, tenant): + raise NotImplementedError() + + def update_tenant(self, tenant_id, tenant): + raise NotImplementedError() + + def delete_tenant(self, tenant_id, tenant): + raise NotImplementedError() + + # metadata crud + def create_metadata(self, user_id, tenant_id, metadata): + raise NotImplementedError() + + def update_metadata(self, user_id, tenant_id, metadata): + raise NotImplementedError() + + def delete_metadata(self, user_id, tenant_id, metadata): + raise NotImplementedError() + + # role crud + def create_role(self, role_id, role): + raise NotImplementedError() + + def update_role(self, role_id, role): + raise NotImplementedError() + + def delete_role(self, role_id): + raise NotImplementedError() + + class PublicRouter(wsgi.ComposableRouter): def add_routes(self, mapper): tenant_controller = TenantController() From 7b4c26d603d3dc03578f7e2043d20660054e8a05 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 17 Jan 2012 21:03:27 -0800 Subject: [PATCH 209/334] add (failing) tests for scoping ec2 crud --- tests/default_fixtures.py | 1 + tests/test_keystoneclient.py | 55 +++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 786becd1f6..82a0d968c5 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -5,6 +5,7 @@ TENANTS = [ USERS = [ {'id': 'foo', 'name': 'FOO', 'password': 'foo2', 'tenants': ['bar',]}, + {'id': 'boo', 'name': 'BOO', 'password': 'boo2', 'tenants': ['baz',]}, ] METADATA = [ diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 3eeed6f61b..caab57efb5 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -65,13 +65,17 @@ class KcMasterTestCase(CompatTestCase): CONF(config_files=[test.etcdir('keystone.conf'), test.testsdir('test_overrides.conf')]) - def foo_client(self): - return self._client(username='FOO', - password='foo2', - tenant_name='BAR') + def get_client(self, username='FOO'): + users = [u for u in default_fixtures.USERS if u['name'] == username] + self.assertEquals(1, len(users)) + user = users[0] + + return self._client(username=user['name'], + password=user['password'], + tenant_id=user['tenants'][0]) def test_authenticate_tenant_name_and_tenants(self): - client = self.foo_client() + client = self.get_client() tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) @@ -84,21 +88,21 @@ class KcMasterTestCase(CompatTestCase): self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_token_no_tenant(self): - client = self.foo_client() + client = self.get_client() token = client.auth_token token_client = self._client(token=token) tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_token_tenant_id(self): - client = self.foo_client() + client = self.get_client() token = client.auth_token token_client = self._client(token=token, tenant_id='bar') tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_token_tenant_name(self): - client = self.foo_client() + client = self.get_client() token = client.auth_token token_client = self._client(token=token, tenant_name='BAR') tenants = client.tenants.list() @@ -106,7 +110,7 @@ class KcMasterTestCase(CompatTestCase): # TODO(termie): I'm not really sure that this is testing much def test_endpoints(self): - client = self.foo_client() + client = self.get_client() token = client.auth_token endpoints = client.tokens.endpoints(token) @@ -119,7 +123,7 @@ class KcMasterTestCase(CompatTestCase): from keystoneclient import exceptions as client_exceptions test_tenant = 'new_tenant' - client = self.foo_client() + client = self.get_client() tenant = client.tenants.create(test_tenant, description="My new tenant!", enabled=True) @@ -142,12 +146,12 @@ class KcMasterTestCase(CompatTestCase): tenant.id) def test_tenant_list(self): - client = self.foo_client() + client = self.get_client() tenants = client.tenants.list() self.assertEquals(len(tenants), 1) def test_tenant_add_and_remove_user(self): - client = self.foo_client() + client = self.get_client() client.roles.add_user_to_tenant(self.tenant_baz['id'], self.user_foo['id'], self.role_useless['id']) @@ -177,7 +181,7 @@ class KcMasterTestCase(CompatTestCase): from keystoneclient import exceptions as client_exceptions test_user = 'new_user' - client = self.foo_client() + client = self.get_client() user = client.users.create(test_user, 'password', 'user1@test.com') self.assertEquals(user.name, test_user) @@ -203,12 +207,12 @@ class KcMasterTestCase(CompatTestCase): user.id) def test_user_list(self): - client = self.foo_client() + client = self.get_client() users = client.users.list() self.assertTrue(len(users) > 0) def test_role_get(self): - client = self.foo_client() + client = self.get_client() role = client.roles.get('keystone_admin') self.assertEquals(role.id, 'keystone_admin') @@ -216,7 +220,7 @@ class KcMasterTestCase(CompatTestCase): from keystoneclient import exceptions as client_exceptions test_role = 'new_role' - client = self.foo_client() + client = self.get_client() role = client.roles.create(test_role) self.assertEquals(role.name, test_role) @@ -229,20 +233,20 @@ class KcMasterTestCase(CompatTestCase): test_role) def test_role_list(self): - client = self.foo_client() + client = self.get_client() roles = client.roles.list() # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(roles) > 0) def test_roles_get_by_user(self): - client = self.foo_client() + client = self.get_client() roles = client.roles.get_user_role_refs('foo') self.assertTrue(len(roles) > 0) - def test_ec2_credential_creation(self): + def test_ec2_credential_crud(self): from keystoneclient import exceptions as client_exceptions - client = self.foo_client() + client = self.get_client() creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) @@ -259,11 +263,18 @@ class KcMasterTestCase(CompatTestCase): creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) + def test_ec2_credentials_scoping(self): + from keystoneclient import exceptions as client_exceptions + + boo = self.get_client('BOO') + self.assertRaises(client_exceptions.Unauthorized, boo.ec2.list, + self.user_foo['id']) + def test_service_create_and_delete(self): from keystoneclient import exceptions as client_exceptions test_service = 'new_service' - client = self.foo_client() + client = self.get_client() service = client.services.create(test_service, 'test', 'test') self.assertEquals(service.name, test_service) @@ -275,7 +286,7 @@ class KcMasterTestCase(CompatTestCase): service.id) def test_service_list(self): - client = self.foo_client() + client = self.get_client() test_service = 'new_service' service = client.services.create(test_service, 'test', 'test') services = client.services.list() From f28a03cf2094d8ba5b64dfea7554afce3ec309a3 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 17 Jan 2012 21:39:01 -0800 Subject: [PATCH 210/334] add METADATA for boo --- tests/default_fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 82a0d968c5..990e241a37 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -10,6 +10,7 @@ USERS = [ METADATA = [ {'user_id': 'foo', 'tenant_id': 'bar', 'extra': 'extra'}, + {'user_id': 'boo', 'tenant_id': 'baz', 'extra': 'extra'}, ] ROLES = [ From 88b0a4bbd1353933614270b90477901a18d6ce7b Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 17 Jan 2012 21:48:31 -0800 Subject: [PATCH 211/334] more failing ec2 tests --- tests/test_keystoneclient.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index caab57efb5..217bb2a77c 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -263,13 +263,37 @@ class KcMasterTestCase(CompatTestCase): creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) - def test_ec2_credentials_scoping(self): + def test_ec2_credentials_scoping_list(self): from keystoneclient import exceptions as client_exceptions boo = self.get_client('BOO') self.assertRaises(client_exceptions.Unauthorized, boo.ec2.list, self.user_foo['id']) + def test_ec2_credentials_scoping_get(self): + from keystoneclient import exceptions as client_exceptions + + foo = self.get_client() + cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) + + boo = self.get_client('BOO') + self.assertRaises(client_exceptions.Unauthorized, boo.ec2.get, + self.user_foo['id'], cred.access) + + foo.ec2.delete(self.user_foo['id'], cred.access) + + def test_ec2_credentials_scoping_delete(self): + from keystoneclient import exceptions as client_exceptions + + foo = self.get_client() + cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) + + boo = self.get_client('BOO') + self.assertRaises(client_exceptions.Unauthorized, boo.ec2.delete, + self.user_foo['id'], cred.access) + + foo.ec2.delete(self.user_foo['id'], cred.access) + def test_service_create_and_delete(self): from keystoneclient import exceptions as client_exceptions From e567fb91464798d11005dc9a7c53e8dce5726ac3 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 18 Jan 2012 11:23:22 -0800 Subject: [PATCH 212/334] use the sql backend for ec2 tests --- tests/backend_sql.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf index 30e8f6f597..a29874870c 100644 --- a/tests/backend_sql.conf +++ b/tests/backend_sql.conf @@ -7,3 +7,6 @@ pool_timeout = 200 [identity] driver = keystone.identity.backends.sql.Identity + +[ec2] +driver = keystone.contrib.ec2.backends.SqlEc2 From c1fe99854c4a4082d377de07d1c925abcddb01a9 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 18 Jan 2012 11:23:43 -0800 Subject: [PATCH 213/334] rename ec2 tests to be more explicit --- tests/test_keystoneclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 217bb2a77c..024edcb1a7 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -263,14 +263,14 @@ class KcMasterTestCase(CompatTestCase): creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) - def test_ec2_credentials_scoping_list(self): + def test_ec2_credentials_list_unauthorized_user(self): from keystoneclient import exceptions as client_exceptions boo = self.get_client('BOO') self.assertRaises(client_exceptions.Unauthorized, boo.ec2.list, self.user_foo['id']) - def test_ec2_credentials_scoping_get(self): + def test_ec2_credentials_get_unauthorized_user(self): from keystoneclient import exceptions as client_exceptions foo = self.get_client() @@ -282,7 +282,7 @@ class KcMasterTestCase(CompatTestCase): foo.ec2.delete(self.user_foo['id'], cred.access) - def test_ec2_credentials_scoping_delete(self): + def test_ec2_credentials_delete_unauthorized_user(self): from keystoneclient import exceptions as client_exceptions foo = self.get_client() From cbc1558ea676456f163b3e060a781cfe21f9932b Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 15:44:48 -0800 Subject: [PATCH 214/334] update how user is specified in tests --- tests/default_fixtures.py | 4 ++-- tests/test_keystoneclient.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 990e241a37..22ac95f0bf 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -5,12 +5,12 @@ TENANTS = [ USERS = [ {'id': 'foo', 'name': 'FOO', 'password': 'foo2', 'tenants': ['bar',]}, - {'id': 'boo', 'name': 'BOO', 'password': 'boo2', 'tenants': ['baz',]}, + {'id': 'two', 'name': 'TWO', 'password': 'two2', 'tenants': ['baz',]}, ] METADATA = [ {'user_id': 'foo', 'tenant_id': 'bar', 'extra': 'extra'}, - {'user_id': 'boo', 'tenant_id': 'baz', 'extra': 'extra'}, + {'user_id': 'two', 'tenant_id': 'baz', 'extra': 'extra'}, ] ROLES = [ diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 024edcb1a7..fb1e25d953 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -65,14 +65,13 @@ class KcMasterTestCase(CompatTestCase): CONF(config_files=[test.etcdir('keystone.conf'), test.testsdir('test_overrides.conf')]) - def get_client(self, username='FOO'): - users = [u for u in default_fixtures.USERS if u['name'] == username] - self.assertEquals(1, len(users)) - user = users[0] + def get_client(self, user_ref=None): + if user_ref is None: + user_ref = self.user_foo - return self._client(username=user['name'], - password=user['password'], - tenant_id=user['tenants'][0]) + return self._client(username=user_ref['name'], + password=user_ref['password'], + tenant_id=user_ref['tenants'][0]) def test_authenticate_tenant_name_and_tenants(self): client = self.get_client() @@ -266,8 +265,8 @@ class KcMasterTestCase(CompatTestCase): def test_ec2_credentials_list_unauthorized_user(self): from keystoneclient import exceptions as client_exceptions - boo = self.get_client('BOO') - self.assertRaises(client_exceptions.Unauthorized, boo.ec2.list, + two = self.get_client(self.user_two) + self.assertRaises(client_exceptions.Unauthorized, two.ec2.list, self.user_foo['id']) def test_ec2_credentials_get_unauthorized_user(self): @@ -276,10 +275,10 @@ class KcMasterTestCase(CompatTestCase): foo = self.get_client() cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) - boo = self.get_client('BOO') - self.assertRaises(client_exceptions.Unauthorized, boo.ec2.get, + two = self.get_client(self.user_two) + self.assertRaises(client_exceptions.Unauthorized, two.ec2.get, self.user_foo['id'], cred.access) - + foo.ec2.delete(self.user_foo['id'], cred.access) def test_ec2_credentials_delete_unauthorized_user(self): @@ -288,8 +287,8 @@ class KcMasterTestCase(CompatTestCase): foo = self.get_client() cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) - boo = self.get_client('BOO') - self.assertRaises(client_exceptions.Unauthorized, boo.ec2.delete, + two = self.get_client(self.user_two) + self.assertRaises(client_exceptions.Unauthorized, two.ec2.delete, self.user_foo['id'], cred.access) foo.ec2.delete(self.user_foo['id'], cred.access) From 21cfcfc38e0e605197405cff73d37b0fb5924a64 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 15:48:28 -0800 Subject: [PATCH 215/334] get_client lets you send user and tenant --- tests/test_keystoneclient.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index fb1e25d953..a31d8adb6c 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -65,13 +65,19 @@ class KcMasterTestCase(CompatTestCase): CONF(config_files=[test.etcdir('keystone.conf'), test.testsdir('test_overrides.conf')]) - def get_client(self, user_ref=None): + def get_client(self, user_ref=None, tenant_ref=None): if user_ref is None: user_ref = self.user_foo + if tenant_ref is None: + for user in default_fixtures.USERS: + if user['id'] == user_ref['id']: + tenant_id = user['tenants'][0] + else: + tenant_id = tenant_ref['id'] return self._client(username=user_ref['name'], password=user_ref['password'], - tenant_id=user_ref['tenants'][0]) + tenant_id=tenant_id) def test_authenticate_tenant_name_and_tenants(self): client = self.get_client() From ecabdd1a7068dadb5603ddbaa3bb38615916d103 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 15:58:30 -0800 Subject: [PATCH 216/334] fix ec2 sql config --- tests/backend_sql.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf index a29874870c..0fec0488a4 100644 --- a/tests/backend_sql.conf +++ b/tests/backend_sql.conf @@ -9,4 +9,4 @@ pool_timeout = 200 driver = keystone.identity.backends.sql.Identity [ec2] -driver = keystone.contrib.ec2.backends.SqlEc2 +driver = keystone.contrib.ec2.backends.sql.Ec2 From 71faa9f0e11a6645f2b6596d325695057a508ba6 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 16:11:31 -0800 Subject: [PATCH 217/334] remove duplicate pycli from pip-requires --- tools/pip-requires | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pip-requires b/tools/pip-requires index c9215f11fb..062e94ee5d 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -7,6 +7,5 @@ routes pycli sqlalchemy sqlalchemy-migrate -pycli -e git+https://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient From 0df93ebd19eae6e44d12572d7924a98ef2d6b022 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 16:25:26 -0800 Subject: [PATCH 218/334] use token_client in token tests --- tests/test_keystoneclient.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index a31d8adb6c..1efe550b42 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -96,21 +96,21 @@ class KcMasterTestCase(CompatTestCase): client = self.get_client() token = client.auth_token token_client = self._client(token=token) - tenants = client.tenants.list() + tenants = token_client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_token_tenant_id(self): client = self.get_client() token = client.auth_token token_client = self._client(token=token, tenant_id='bar') - tenants = client.tenants.list() + tenants = token_client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_token_tenant_name(self): client = self.get_client() token = client.auth_token token_client = self._client(token=token, tenant_name='BAR') - tenants = client.tenants.list() + tenants = token_client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) # TODO(termie): I'm not really sure that this is testing much @@ -173,7 +173,6 @@ class KcMasterTestCase(CompatTestCase): # use python's scope fall through to leave roleref_ref set break - client.roles.remove_user_from_tenant(self.tenant_baz['id'], self.user_foo['id'], roleref_ref.id) @@ -296,7 +295,7 @@ class KcMasterTestCase(CompatTestCase): two = self.get_client(self.user_two) self.assertRaises(client_exceptions.Unauthorized, two.ec2.delete, self.user_foo['id'], cred.access) - + foo.ec2.delete(self.user_foo['id'], cred.access) def test_service_create_and_delete(self): From ffeb0e558ca1108df02c53c9170e73020e57e67c Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Thu, 19 Jan 2012 01:52:44 +0000 Subject: [PATCH 219/334] doctry --- docs/source/conf.py | 29 ++++++++++++++++++++++------- setup.py | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c18c5ff69..11c8adb0f7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -26,10 +27,19 @@ sys.path.insert(0, os.path.abspath('../..')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage'] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.todo', +# 'sphinx.ect.intersphinx', + 'sphinx.ext.coverage'] + +todo_include_todos = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = [] +if os.getenv('HUDSON_PUBLISH_DOCS'): + templates_path = ['_ga', '_templates'] +else: + templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' @@ -79,20 +89,21 @@ exclude_patterns = [] # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +show_authors = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +modindex_common_prefix = ['keystone.'] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme_path = ["."] +html_theme = '_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -121,7 +132,7 @@ html_theme = 'default' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['_static', 'images'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -245,3 +256,7 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. #intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'python': ('http://docs.python.org/', None), + 'nova': ('http://nova.openstack.org', None), + 'swift': ('http://swift.openstack.org', None), + 'glance': ('http://glance.openstack.org', None)} diff --git a/setup.py b/setup.py index c8a3d75019..fa418f3107 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,40 @@ +import os +import subprocess + from setuptools import setup, find_packages +# If Sphinx is installed on the box running setup.py, +# enable setup.py to build the documentation, otherwise, +# just ignore it +cmdclass = {} +try: + from sphinx.setup_command import BuildDoc + + class local_BuildDoc(BuildDoc): + def run(self): + base_dir = os.path.dirname(os.path.abspath(__file__)) + subprocess.Popen(["python", "generate_autodoc_index.py"], + cwd=os.path.join(base_dir, "docs")).communicate() + for builder in ['html', 'man']: + self.builder = builder + self.finalize_options() + BuildDoc.run(self) + cmdclass['build_sphinx'] = local_BuildDoc +except: + # unable to import sphinx, politely skip past... + pass + + setup(name='keystone', version='2012.1', description="Authentication service for OpenStack", + license='Apache License (2.0)', author='OpenStack, LLC.', author_email='openstack@lists.launchpad.net', url='http://www.openstack.org', packages=find_packages(exclude=['test', 'bin']), scripts=['bin/keystone', 'bin/keystone-manage'], zip_safe=False, + cmdclass=cmdclass, install_requires=['setuptools'], ) From 3d2bb3a355274c6ce013fb03accb4811d467939e Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 17:15:03 -0800 Subject: [PATCH 220/334] test login fails with invalid password or disabled user --- tests/test_keystoneclient.py | 38 ++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 1efe550b42..c56416eaf0 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -181,16 +181,29 @@ class KcMasterTestCase(CompatTestCase): self.assert_(self.tenant_baz['id'] not in [x.id for x in tenant_refs]) + def test_invalid_password(self): + from keystoneclient import exceptions as client_exceptions + + good_client = self._client(username=self.user_foo['name'], + password=self.user_foo['password']) + good_client.tenants.list() + + self.assertRaises(client_exceptions.Unauthorized, + self._client, + username=self.user_foo['name'], + password='invalid') + + def test_user_create_update_delete(self): from keystoneclient import exceptions as client_exceptions - test_user = 'new_user' + test_username = 'new_user' client = self.get_client() - user = client.users.create(test_user, 'password', 'user1@test.com') - self.assertEquals(user.name, test_user) + user = client.users.create(test_username, 'password', 'user1@test.com') + self.assertEquals(user.name, test_username) user = client.users.get(user.id) - self.assertEquals(user.name, test_user) + self.assertEquals(user.name, test_username) user = client.users.update_email(user, 'user2@test.com') self.assertEquals(user.email, 'user2@test.com') @@ -200,11 +213,22 @@ class KcMasterTestCase(CompatTestCase): user = client.users.get(user.id) self.assertFalse(user.enabled) - # TODO(devcamcar): How to assert this succeeded? + # TODO(ja): test that you can't login + self.assertRaises(client_exceptions.Unauthorized, + self._client, + username=test_username, + password='password') + client.users.update_enabled(user, True) + user = client.users.update_password(user, 'password2') - # TODO(devcamcar): How to assert this succeeded? + test_client = self._client(username=test_username, + password='password2') + user = client.users.update_tenant(user, 'bar') + # TODO(ja): once keystonelight supports default tenant + # when you login without specifying tenant, the + # token should be scoped to tenant 'bar' client.users.delete(user.id) self.assertRaises(client_exceptions.NotFound, client.users.get, @@ -248,8 +272,6 @@ class KcMasterTestCase(CompatTestCase): self.assertTrue(len(roles) > 0) def test_ec2_credential_crud(self): - from keystoneclient import exceptions as client_exceptions - client = self.get_client() creds = client.ec2.list(self.user_foo['id']) self.assertEquals(creds, []) From f40198dece0b3729e6eb70abaa972bd73ee827da Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Thu, 19 Jan 2012 17:30:27 -0800 Subject: [PATCH 221/334] shimming in basics from original keystone --- docs/generate_autodoc_index.py | 76 +++++ docs/source/_static/basic.css | 416 ++++++++++++++++++++++++++++ docs/source/_static/default.css | 230 +++++++++++++++ docs/source/_static/jquery.tweet.js | 154 ++++++++++ docs/source/_static/tweaks.css | 65 +++++ docs/source/_templates/.placeholder | 0 docs/source/_theme/layout.html | 86 ++++++ docs/source/_theme/theme.conf | 5 + docs/source/community.rst | 93 +++++++ docs/source/conf.py | 12 + docs/source/developing.rst | 135 +++++++++ docs/source/index.rst | 58 +++- docs/source/man/keystone-manage.rst | 192 +++++++++++++ docs/source/man/keystone.rst | 90 ++++++ docs/source/setup.rst | 151 ++++++++++ docs/source/testing.rst | 77 +++++ 16 files changed, 1832 insertions(+), 8 deletions(-) create mode 100755 docs/generate_autodoc_index.py create mode 100644 docs/source/_static/basic.css create mode 100644 docs/source/_static/default.css create mode 100644 docs/source/_static/jquery.tweet.js create mode 100644 docs/source/_static/tweaks.css create mode 100644 docs/source/_templates/.placeholder create mode 100644 docs/source/_theme/layout.html create mode 100644 docs/source/_theme/theme.conf create mode 100644 docs/source/community.rst create mode 100644 docs/source/developing.rst create mode 100644 docs/source/man/keystone-manage.rst create mode 100644 docs/source/man/keystone.rst create mode 100644 docs/source/setup.rst create mode 100644 docs/source/testing.rst diff --git a/docs/generate_autodoc_index.py b/docs/generate_autodoc_index.py new file mode 100755 index 0000000000..993369b028 --- /dev/null +++ b/docs/generate_autodoc_index.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +"""Generates files for sphinx documentation using a simple Autodoc based +template. + +To use, just run as a script: + $ python doc/generate_autodoc_index.py +""" + +import os + + +base_dir = os.path.dirname(os.path.abspath(__file__)) +RSTDIR=os.path.join(base_dir, "source", "sourcecode") +SOURCEDIR=os.path.join(base_dir, "..") + +# Exclude these modules from the autodoc results +EXCLUDE_MODULES = ['keystone.backends.sqlalchemy.migrate_repo'] + +def in_exclude_list(module_name): + """Compares a module to the list of excluded modules + + Returns true if the provided module resides in or matches + an excluded module, false otherwise. + """ + for excluded_module in EXCLUDE_MODULES: + if module_name.startswith(excluded_module): + return True + return False + +def find_autodoc_modules(module_name, sourcedir): + """returns a list of modules in the SOURCE directory""" + modlist = [] + os.chdir(os.path.join(sourcedir, module_name)) + for root, dirs, files in os.walk("."): + for filename in files: + if filename.endswith(".py"): + # root = ./keystone/test/unit + # filename = base.py + elements = root.split(os.path.sep) + # replace the leading "." with the module name + elements[0] = module_name + # and get the base module name + base, extension = os.path.splitext(filename) + if not (base == "__init__"): + elements.append(base) + result = (".".join(elements)) + if not in_exclude_list(result): + modlist.append(result) + return modlist + +if not(os.path.exists(RSTDIR)): + os.mkdir(RSTDIR) + +INDEXOUT = open("%s/autoindex.rst" % RSTDIR, "w") +INDEXOUT.write("Source Code Index\n") +INDEXOUT.write("=================\n") +INDEXOUT.write(".. toctree::\n") +INDEXOUT.write(" :maxdepth: 1\n") +INDEXOUT.write("\n") + +for module in find_autodoc_modules('keystone', SOURCEDIR): + generated_file = "%s/%s.rst" % (RSTDIR, module) + + INDEXOUT.write(" %s\n" % module) + FILEOUT = open(generated_file, "w") + FILEOUT.write("The :mod:`%s` Module\n" % module) + FILEOUT.write("==============================" + "==============================" + "==============================\n") + FILEOUT.write(".. automodule:: %s\n" % module) + FILEOUT.write(" :members:\n") + FILEOUT.write(" :undoc-members:\n") + FILEOUT.write(" :show-inheritance:\n") + FILEOUT.close() + +INDEXOUT.close() diff --git a/docs/source/_static/basic.css b/docs/source/_static/basic.css new file mode 100644 index 0000000000..d909ce37c7 --- /dev/null +++ b/docs/source/_static/basic.css @@ -0,0 +1,416 @@ +/** + * Sphinx stylesheet -- basic theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +img { + border: 0; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + border: 0; + border-collapse: collapse; +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 0; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +/* -- other body styles ----------------------------------------------------- */ + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff --git a/docs/source/_static/default.css b/docs/source/_static/default.css new file mode 100644 index 0000000000..c8091ecb4d --- /dev/null +++ b/docs/source/_static/default.css @@ -0,0 +1,230 @@ +/** + * Sphinx stylesheet -- default theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: sans-serif; + font-size: 100%; + background-color: #11303d; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + background-color: #1c4e63; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 230px; +} + +div.body { + background-color: #ffffff; + color: #000000; + padding: 0 20px 30px 20px; +} + +div.footer { + color: #ffffff; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #ffffff; + text-decoration: underline; +} + +div.related { + background-color: #133f52; + line-height: 30px; + color: #ffffff; +} + +div.related a { + color: #ffffff; +} + +div.sphinxsidebar { +} + +div.sphinxsidebar h3 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h3 a { + color: #ffffff; +} + +div.sphinxsidebar h4 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: #ffffff; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: #ffffff; +} + +div.sphinxsidebar a { + color: #98dbcc; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #355f7c; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: 'Trebuchet MS', sans-serif; + background-color: #f2f2f2; + font-weight: normal; + color: #20435c; + border-bottom: 1px solid #ccc; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 5px; + background-color: #eeffcc; + color: #333333; + line-height: 120%; + border: 1px solid #ac9; + border-left: none; + border-right: none; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} diff --git a/docs/source/_static/jquery.tweet.js b/docs/source/_static/jquery.tweet.js new file mode 100644 index 0000000000..c93fea8768 --- /dev/null +++ b/docs/source/_static/jquery.tweet.js @@ -0,0 +1,154 @@ +(function($) { + + $.fn.tweet = function(o){ + var s = { + username: ["seaofclouds"], // [string] required, unless you want to display our tweets. :) it can be an array, just do ["username1","username2","etc"] + list: null, //[string] optional name of list belonging to username + avatar_size: null, // [integer] height and width of avatar if displayed (48px max) + count: 3, // [integer] how many tweets to display? + intro_text: null, // [string] do you want text BEFORE your your tweets? + outro_text: null, // [string] do you want text AFTER your tweets? + join_text: null, // [string] optional text in between date and tweet, try setting to "auto" + auto_join_text_default: "i said,", // [string] auto text for non verb: "i said" bullocks + auto_join_text_ed: "i", // [string] auto text for past tense: "i" surfed + auto_join_text_ing: "i am", // [string] auto tense for present tense: "i was" surfing + auto_join_text_reply: "i replied to", // [string] auto tense for replies: "i replied to" @someone "with" + auto_join_text_url: "i was looking at", // [string] auto tense for urls: "i was looking at" http:... + loading_text: null, // [string] optional loading text, displayed while tweets load + query: null // [string] optional search query + }; + + if(o) $.extend(s, o); + + $.fn.extend({ + linkUrl: function() { + var returning = []; + var regexp = /((ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?)/gi; + this.each(function() { + returning.push(this.replace(regexp,"$1")); + }); + return $(returning); + }, + linkUser: function() { + var returning = []; + var regexp = /[\@]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp,"@$1")); + }); + return $(returning); + }, + linkHash: function() { + var returning = []; + var regexp = / [\#]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp, ' #$1')); + }); + return $(returning); + }, + capAwesome: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(awesome)\b/gi, '$1')); + }); + return $(returning); + }, + capEpic: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(epic)\b/gi, '$1')); + }); + return $(returning); + }, + makeHeart: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/(<)+[3]/gi, "♥")); + }); + return $(returning); + } + }); + + function relative_time(time_value) { + var parsed_date = Date.parse(time_value); + var relative_to = (arguments.length > 1) ? arguments[1] : new Date(); + var delta = parseInt((relative_to.getTime() - parsed_date) / 1000); + var pluralize = function (singular, n) { + return '' + n + ' ' + singular + (n == 1 ? '' : 's'); + }; + if(delta < 60) { + return 'less than a minute ago'; + } else if(delta < (45*60)) { + return 'about ' + pluralize("minute", parseInt(delta / 60)) + ' ago'; + } else if(delta < (24*60*60)) { + return 'about ' + pluralize("hour", parseInt(delta / 3600)) + ' ago'; + } else { + return 'about ' + pluralize("day", parseInt(delta / 86400)) + ' ago'; + } + } + + function build_url() { + var proto = ('https:' == document.location.protocol ? 'https:' : 'http:'); + if (s.list) { + return proto+"//api.twitter.com/1/"+s.username[0]+"/lists/"+s.list+"/statuses.json?per_page="+s.count+"&callback=?"; + } else if (s.query == null && s.username.length == 1) { + return proto+'//twitter.com/status/user_timeline/'+s.username[0]+'.json?count='+s.count+'&callback=?'; + } else { + var query = (s.query || 'from:'+s.username.join('%20OR%20from:')); + return proto+'//search.twitter.com/search.json?&q='+query+'&rpp='+s.count+'&callback=?'; + } + } + + return this.each(function(){ + var list = $('
    ').appendTo(this); + var intro = '

    '+s.intro_text+'

    '; + var outro = '

    '+s.outro_text+'

    '; + var loading = $('

    '+s.loading_text+'

    '); + + if(typeof(s.username) == "string"){ + s.username = [s.username]; + } + + if (s.loading_text) $(this).append(loading); + $.getJSON(build_url(), function(data){ + if (s.loading_text) loading.remove(); + if (s.intro_text) list.before(intro); + $.each((data.results || data), function(i,item){ + // auto join text based on verb tense and content + if (s.join_text == "auto") { + if (item.text.match(/^(@([A-Za-z0-9-_]+)) .*/i)) { + var join_text = s.auto_join_text_reply; + } else if (item.text.match(/(^\w+:\/\/[A-Za-z0-9-_]+\.[A-Za-z0-9-_:%&\?\/.=]+) .*/i)) { + var join_text = s.auto_join_text_url; + } else if (item.text.match(/^((\w+ed)|just) .*/im)) { + var join_text = s.auto_join_text_ed; + } else if (item.text.match(/^(\w*ing) .*/i)) { + var join_text = s.auto_join_text_ing; + } else { + var join_text = s.auto_join_text_default; + } + } else { + var join_text = s.join_text; + }; + + var from_user = item.from_user || item.user.screen_name; + var profile_image_url = item.profile_image_url || item.user.profile_image_url; + var join_template = ' '+join_text+' '; + var join = ((s.join_text) ? join_template : ' '); + var avatar_template = ''+from_user+'\'s avatar'; + var avatar = (s.avatar_size ? avatar_template : ''); + var date = ''+relative_time(item.created_at)+''; + var text = '' +$([item.text]).linkUrl().linkUser().linkHash().makeHeart().capAwesome().capEpic()[0]+ ''; + + // until we create a template option, arrange the items below to alter a tweet's display. + list.append('
  • ' + avatar + date + join + text + '
    • '); + + list.children('li:first').addClass('tweet_first'); + list.children('li:odd').addClass('tweet_even'); + list.children('li:even').addClass('tweet_odd'); + }); + if (s.outro_text) list.after(outro); + }); + + }); + }; +})(jQuery); \ No newline at end of file diff --git a/docs/source/_static/tweaks.css b/docs/source/_static/tweaks.css new file mode 100644 index 0000000000..16cd6e76e2 --- /dev/null +++ b/docs/source/_static/tweaks.css @@ -0,0 +1,65 @@ +ul.todo_list { + list-style-type: none; + margin: 0; + padding: 0; +} + +ul.todo_list li { + display: block; + margin: 0; + padding: 7px 0; + border-top: 1px solid #eee; +} + +ul.todo_list li p { + display: inline; +} + +ul.todo_list li p.link { + font-weight: bold; +} + +ul.todo_list li p.details { + font-style: italic; +} + +ul.todo_list li { +} + +div.admonition { + border: 1px solid #8F1000; +} + +div.admonition p.admonition-title { + background-color: #8F1000; + border-bottom: 1px solid #8E8E8E; +} + +a { + color: #CF2F19; +} + +div.related ul li a { + color: #CF2F19; +} + +div.sphinxsidebar h4 { + background-color:#8E8E8E; + border:1px solid #255E6E; + color:white; + font-size:1em; + margin:1em 0 0.5em; + padding:0.1em 0 0.1em 0.5em; +} + +em { + font-style: normal; +} + +table.docutils { + font-size: 11px; +} + +a tt { + color:#CF2F19; +} \ No newline at end of file diff --git a/docs/source/_templates/.placeholder b/docs/source/_templates/.placeholder new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/source/_theme/layout.html b/docs/source/_theme/layout.html new file mode 100644 index 0000000000..e3eb54b712 --- /dev/null +++ b/docs/source/_theme/layout.html @@ -0,0 +1,86 @@ +{% extends "sphinxdoc/layout.html" %} +{% set css_files = css_files + ['_static/tweaks.css'] %} +{% set script_files = script_files + ['_static/jquery.tweet.js'] %} +{% block extrahead %} + +{% endblock %} + +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
    +
    + {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

    {{ _('Table Of Contents') }}

    + {{ toc }} + {%- endif %} + {%- endblock %} + {%- block sidebarrel %} + {%- if prev %} +

    {{ _('Previous topic') }}

    +

    {{ prev.title }}

    + {%- endif %} + {%- if next %} +

    {{ _('Next topic') }}

    +

    {{ next.title }}

    + {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

    {{ _('This Page') }}

    + + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + + {%- if pagename == "index" %} +

    {{ _('Twitter Feed') }}

    + + {%- endif %} + + + {%- endblock %} +
    +
    + {%- endif %}{% endif %} +{%- endmacro %} diff --git a/docs/source/_theme/theme.conf b/docs/source/_theme/theme.conf new file mode 100644 index 0000000000..e039fe01f9 --- /dev/null +++ b/docs/source/_theme/theme.conf @@ -0,0 +1,5 @@ +[theme] +inherit = sphinxdoc +stylesheet = sphinxdoc.css +pygments_style = friendly + diff --git a/docs/source/community.rst b/docs/source/community.rst new file mode 100644 index 0000000000..bbad242147 --- /dev/null +++ b/docs/source/community.rst @@ -0,0 +1,93 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +================ +Getting Involved +================ + +The OpenStack community is a very friendly group and there are places online to join in with the +community. Feel free to ask questions. This document points you to some of the places where you can +communicate with people. + +How to Join the Community +========================= + +Our community welcomes all people interested in open source cloud computing, and there are no formal +membership requirements. The best way to join the community is to talk with others online or at a meetup +and offer contributions through Launchpad_, the wiki_, or blogs. We welcome all types of contributions, +from blueprint designs to documentation to testing to deployment scripts. + +.. _Launchpad: https://launchpad.net/keystone +.. _wiki: http://wiki.openstack.org/ + + + +Contributing Code +----------------- + +To contribute code, sign up for a Launchpad account and sign a contributor license agreement, +available on the ``_. Once the CLA is signed you +can contribute code through the Gerrit version control system which is related to your Launchpad account. + +To contribute tests, docs, code, etc, refer to our `Gerrit-Jenkins-Github Workflow`_. + +.. _`Gerrit-Jenkins-Github Workflow`: http://wiki.openstack.org/GerritJenkinsGithub + + +#openstack on Freenode IRC Network +---------------------------------- + +There is a very active chat channel at ``_. This +is usually the best place to ask questions and find your way around. IRC stands for Internet Relay +Chat and it is a way to chat online in real time. You can also ask a question and come back to the +log files to read the answer later. Logs for the #openstack IRC channel are stored at +``_. + +OpenStack Wiki +-------------- + +The wiki is a living source of knowledge. It is edited by the community, and +has collections of links and other sources of information. Typically the pages are a good place +to write drafts for specs or documentation, describe a blueprint, or collaborate with others. + +`OpenStack Wiki `_ + +Keystone on Launchpad +--------------------- + +Launchpad is a code hosting service that hosts the Keystone source code. From +Launchpad you can report bugs, ask questions, and register blueprints (feature requests). + +* `Launchpad Keystone Page `_ + +OpenStack Blog +-------------- + +The OpenStack blog includes a weekly newsletter that aggregates OpenStack news +from around the internet, as well as providing inside information on upcoming +events and posts from OpenStack contributors. + +`OpenStack Blog `_ + +See also: `Planet OpenStack `_, aggregating blogs +about OpenStack from around the internet into a single feed. If you'd like to contribute to this blog +aggregation with your blog posts, there are instructions for `adding your blog `_. + +Twitter +------- + +Because all the cool kids do it: `@openstack `_. Also follow the +`#openstack `_ tag for relevant tweets. diff --git a/docs/source/conf.py b/docs/source/conf.py index 11c8adb0f7..6351572659 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -97,6 +97,18 @@ pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. modindex_common_prefix = ['keystone.'] +# -- Options for man page output -------------------------------------------- + +# Grouping the document tree for man pages. +# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual' + +man_pages = [ + ('man/keystone-manage', 'keystone-manage', u'Keystone Management Utility', + [u'OpenStack'], 1), + ('man/keystone', 'keystone', u'Keystone Startup Command', + [u'OpenStack'], 1), + ] + # -- Options for HTML output --------------------------------------------------- diff --git a/docs/source/developing.rst b/docs/source/developing.rst new file mode 100644 index 0000000000..acd2273068 --- /dev/null +++ b/docs/source/developing.rst @@ -0,0 +1,135 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +======================== +Developing with Keystone +======================== + +Get your development environment set up according to :doc:`setup`. + +Running a development instance +============================== + +Setting up a virtualenv +----------------------- + +We recommend establishing a virtualenv to run keystone within. To establish +this environment, use the command:: + + $ python tools/install_venv.py + +This will create a local virtual environment in the directory ``.venv``. +Once created, you can activate this virtualenv for your current shell using:: + + $ source .venv/bin/activate + +The virtual environment can be disabled using the command:: + + $ deactivate + +You can also use ``tools\with_venv.sh`` to prefix commands so that they run +within the virtual environment. For more information on virtual environments, +see virtualenv_. + +.. _virtualenv: http://www.virtualenv.org/ + +Running Keystone +---------------- + +To run the keystone Admin and API server instances, use:: + + $ tools/with_venv.sh bin/keystone + +Running a demo service that uses Keystone +----------------------------------------- + +To run client demo (with all auth middleware running locally on sample service):: + + $ tools/with_venv.sh examples/echo/bin/echod + +which spins up a simple "echo" service on port 8090. To use a simple echo client:: + + $ python examples/echo/echo_client.py + +Interacting with Keystone +========================= + +You can interact with Keystone through the command line using :doc:`man/keystone-manage` +which allows you to establish tenants, users, etc. + +You can also interact with Keystone through it's REST API. There is a python +keystone client library python-keystoneclient_ which interacts exclusively through +the REST API. + +.. _python-keystoneclient: https://github.com/4P/python-keystoneclient + +The easiest way to establish some base information in Keystone to interact with is +to invoke:: + + $ tools/with_venv.sh bin/sampledata + +You can see the details of what that creates in ``keystone/test/sampledata.py`` + +Enabling debugging middleware +----------------------------- + +You can enable a huge amount of additional data (debugging information) about +the request and repsonse objects flowing through Keystone using the debugging +WSGI middleware. + +To enable this, just modify the pipelines in ``etc/keystone.conf``, from:: + + [pipeline:admin] + pipeline = + urlnormalizer + admin_api + + [pipeline:keystone-legacy-auth] + pipeline = + urlnormalizer + legacy_auth + d5_compat + service_api + +... to:: + + [pipeline:admin] + pipeline = + debug + urlnormalizer + d5_compat + admin_api + + [pipeline:keystone-legacy-auth] + pipeline = + debug + urlnormalizer + legacy_auth + d5_compat + service_api + +Two simple and easy debugging tools are using the ``-d`` when you start keystone:: + + $ ./keystone -d + +and the `--trace-calls` flag:: + + $ ./keystone -trace-calls + +The ``-d`` flag outputs debug information to the console. The ``--trace-calls`` flag +outputs extensive, nested trace calls to the console and highlights any errors +in red. + diff --git a/docs/source/index.rst b/docs/source/index.rst index cd7c96a8ff..4270033a80 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,17 +1,59 @@ -.. keystone documentation master file, created by - sphinx-quickstart on Mon Jan 9 12:02:59 2012. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. -Welcome to keystone's documentation! -==================================== + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at -Contents: + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +==================================================== +Welcome to Keystone, the OpenStack Identity Service! +==================================================== + +Keystone is a cloud identity service written in Python, which provides +authentication, authorization, and an OpenStack service catalog. It +implements `OpenStack's Identity API`_. + +This document describes Keystone for contributors of the project, and assumes +that you are already familiar with Keystone from an `end-user perspective`_. + +.. _`OpenStack's Identity API`: http://docs.openstack.org/api/openstack-identity-service/2.0/content/ +.. _`end-user perspective`: http://docs.openstack.org/ + +This documentation is generated by the Sphinx toolkit and lives in the source +tree. Additional documentation on Keystone and other components of OpenStack can +be found on the `OpenStack wiki`_. Also see the :doc:`community` page for +other ways to interact with the community. + +.. _`OpenStack wiki`: http://wiki.openstack.org + +Getting Started +=============== .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + + setup + community + testing +Developers Documentation +======================== +.. toctree:: + :maxdepth: 1 + + developing + architecture + sourcecode/autoindex Indices and tables ================== diff --git a/docs/source/man/keystone-manage.rst b/docs/source/man/keystone-manage.rst new file mode 100644 index 0000000000..9e9304f04d --- /dev/null +++ b/docs/source/man/keystone-manage.rst @@ -0,0 +1,192 @@ +=============== +keystone-manage +=============== + +--------------------------- +Keystone Management Utility +--------------------------- + +:Author: keystone@lists.launchpad.net +:Date: 2010-11-16 +:Copyright: OpenStack LLC +:Version: 0.1.2 +:Manual section: 1 +:Manual group: cloud computing + +SYNOPSIS +======== + + keystone-manage [options] + +DESCRIPTION +=========== + +keystone-manage is the command line tool that interacts with the keystone +service to configure Keystone + +USAGE +===== + + ``keystone-manage [options] type action [additional args]`` + +user +---- + +* **user add** [username] [password] + + adds a user to Keystone's data store + +* **user list** + + lists all users + +* **user disable** [username] + + disables the user *username* + +tenant +------ + +* **tenant add** [tenant_name] + + adds a tenant to Keystone's data store + +* **tenant list** + + lists all users + +* **tenant disable** [tenant_name] + +role +---- + +Roles are used to associated users to tenants. Two roles are defined related +to the Keystone service in it's configuration file :doc:`../keystone.conf` + +* **role add** [role_name] + + adds a role + +* **role list** ([tenant_name]) + + lists all roles, or all roles for tenant, if tenant_name is provided + +* **role grant** [role_name] [username] ([tenant]) + + grants a role to a specific user. Granted globally if tenant_name is not + provided or granted for a specific tenant if tenant_name is provided. + +service +------- + +* **service add** [name] [type] [description] [owner_id] + + adds a service + +* **service list** + + lists all services with id, name, and type + +endpointTemplate +---------------- + +* **endpointTemplate add** [region] [service_name] [public_url] [admin_url] [internal_url] [enabled] [is_global] + + Add a service endpoint for keystone. + + example:: + + keystone-manage endpointTemplates add RegionOne \ + keystone \ + http://keystone_host:5000/v2.0 \ + http://keystone_host:35357/v2.0 \ + http://keystone_host:5000/v2.0 \ + 1 1 + +* **endpointTemplate list** ([tenant_name]) + + lists endpoint templates with service, region, and public_url. Restricted to + tenant endpoints if tenant_name is provided. + +token +----- + +* **token add** [token] [username] [tenant] [expiration] + + adds a token for a given user and tenant with an expiration + +* **token list** + + lists all tokens + +* **token delete** [token] + + deletes the identified token + +endpoint +-------- + +* **endpoint add** [tenant_name] [endpoint_template] + + adds a tenant-specific endpoint + +credentials +----------- + +* **credentials add** [username] [type] [key] [password] ([tenant_name]) + +OPTIONS +======= + + --version show program's version number and exit + -h, --help show this help message and exit + -v, --verbose Print more verbose output + -d, --debug Print debugging output to console + -c PATH, --config-file=PATH Path to the config file to use. When not + specified (the default), we generally look at + the first argument specified to be a config + file, and if that is also missing, we search + standard directories for a config file. + -p BIND_PORT, --port=BIND_PORT, --bind-port=BIND_PORT + specifies port to listen on (default is 5000) + --host=BIND_HOST, --bind-host=BIND_HOST + specifies host address to listen on (default + is all or 0.0.0.0) + -t, --trace-calls Turns on call tracing for troubleshooting + -a PORT, --admin-port=PORT Specifies port for Admin API to listen on + (default is 35357) + +Logging Options: +================ + +The following configuration options are specific to logging +functionality for this program. + + --log-config=PATH If this option is specified, the logging + configuration file specified is used and + overrides any other logging options specified. + Please see the Python logging module + documentation for details on logging + configuration files. + --log-date-format=FORMAT Format string for %(asctime)s in log records. + Default: %Y-%m-%d %H:%M:%S + --log-file=PATH (Optional) Name of log file to output to. If + not set, logging will go to stdout. + --log-dir=LOG_DIR (Optional) The directory to keep log files in + (will be prepended to --logfile) + +FILES +===== + +None + +SEE ALSO +======== + +* `Keystone `__ + +SOURCE +====== + +* Keystone is sourced in GitHub `Keystone `__ +* Keystone bugs are managed at Launchpad `Launchpad Keystone `__ diff --git a/docs/source/man/keystone.rst b/docs/source/man/keystone.rst new file mode 100644 index 0000000000..48e062e436 --- /dev/null +++ b/docs/source/man/keystone.rst @@ -0,0 +1,90 @@ +======== +keystone +======== + +--------------------------- +Keystone Management Utility +--------------------------- + +:Author: keystone@lists.launchpad.net +:Date: 2010-11-16 +:Copyright: OpenStack LLC +:Version: 0.1.2 +:Manual section: 1 +:Manual group: cloud computing + +SYNOPSIS +======== + + keystone [options] + +DESCRIPTION +=========== + +keystone starts both the service and administrative API servers for Keystone. +Use :doc:`keystone-control` to stop/start/restart and manage those services +once started. + +USAGE +===== + + keystone ``keystone [options]`` + +Common Options: +^^^^^^^^^^^^^^^ + --version show program's version number and exit + -h, --help show this help message and exit + +The following configuration options are common to all keystone +programs.:: + + -v, --verbose Print more verbose output + -d, --debug Print debugging output to console + -c PATH, --config-file=PATH Path to the config file to use. When not + specified (the default), we generally look at + the first argument specified to be a config + file, and if that is also missing, we search + standard directories for a config file. + -p BIND_PORT, --port=BIND_PORT, --bind-port=BIND_PORT + specifies port to listen on (default is 5000) + --host=BIND_HOST, --bind-host=BIND_HOST + specifies host address to listen on (default + is all or 0.0.0.0) + -t, --trace-calls Turns on call tracing for troubleshooting + -a PORT, --admin-port=PORT Specifies port for Admin API to listen on + (default is 35357) + +Logging Options: +^^^^^^^^^^^^^^^^ + +The following configuration options are specific to logging +functionality for this program.:: + + --log-config=PATH If this option is specified, the logging + configuration file specified is used and + overrides any other logging options specified. + Please see the Python logging module + documentation for details on logging + configuration files. + --log-date-format=FORMAT Format string for %(asctime)s in log records. + Default: %Y-%m-%d %H:%M:%S + --log-file=PATH (Optional) Name of log file to output to. If + not set, logging will go to stdout. + --log-dir=LOG_DIR (Optional) The directory to keep log files in + (will be prepended to --logfile) + +FILES +===== + +None + +SEE ALSO +======== + +* `Keystone `__ + +SOURCE +====== + +* Keystone is sourced in GitHub `Keystone `__ +* Keystone bugs are managed at Launchpad `Launchpad Keystone `__ diff --git a/docs/source/setup.rst b/docs/source/setup.rst new file mode 100644 index 0000000000..e608f09e4c --- /dev/null +++ b/docs/source/setup.rst @@ -0,0 +1,151 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================================= +Setting up a Keystone development environment +============================================= + +This document describes setting up keystone directly from GitHub_ +for development purposes. + +To install keystone from packaging, refer instead to Keystone's `User Documentation`_. + +.. _GitHub: http://github.com/openstack/keystone +.. _`User Documentation`: http://docs.openstack.org/ + +Prerequisites +============= + +This document assumes you are using: + +- Ubuntu 11.10, Fedora 15, or Mac OS X Lion +- `Python 2.7`_ + +.. _`Python 2.7`: http://www.python.org/ + +And that you have the following tools available on your system: + +- git_ +- setuptools_ +- pip_ + +**Reminder**: If you're successfully using a different platform, or a +different version of the above, please document your configuration here! + +.. _git: http://git-scm.com/ +.. _setuptools: http://pypi.python.org/pypi/setuptools + +Getting the latest code +======================= + +You can clone our latest code from our `Github repository`:: + + $ git clone https://github.com/openstack/keystone.git + +When that is complete, you can:: + + $ cd keystone + +.. _`Github repository`: https://github.com/openstack/keystone + +Installing dependencies +======================= + +Keystone maintains a list of PyPi_ dependencies, designed for use by +pip_. + +.. _PyPi: http://pypi.python.org/ +.. _pip: http://pypi.python.org/pypi/pip + +However, your system *may* need additional dependencies that `pip` (and by +extension, PyPi) cannot satisfy. These dependencies should be installed +prior to using `pip`, and the installation method may vary depending on +your platform. + +Ubuntu 11.10:: + + $ sudo apt-get install python-dev libxml2-dev libxslt1-dev libsasl2-dev libsqlite3-dev libssl-dev libldap2-dev + +Fedora 15:: + + $ sudo yum install python-sqlite2 python-lxml python-greenlet-devel python-ldap + +Mac OS X Lion (requires MacPorts_):: + + $ sudo port install py-ldap + +.. _MacPorts: http://www.macports.org/ + +PyPi Packages +------------- + +Assuming you have any necessary binary packages & header files available +on your system, you can then install PyPi dependencies. + +You may also need to prefix `pip install` with `sudo`, depending on your +environment:: + + # Describe dependencies (including non-PyPi dependencies) + $ cat tools/pip-requires + + # Install all PyPi dependencies (for production, testing, and development) + $ pip install -r tools/pip-requires + +Updating your PYTHONPATH +======================== + +There are a number of methods for getting Keystone into your PYTHON PATH, +the easiest of which is:: + + # Fake-install the project by symlinking Keystone into your Python site-packages + $ python setup.py develop + +You should then be able to `import keystone` from your Python shell +without issue:: + + >>> import keystone.version + >>> + +If you want to check the version of Keystone you are running: + + >>> print keystone.version.version() + 2012.1-dev + + +If you can import keystone successfully, you should be ready to move on to :doc:`testing`. + +Troubleshooting +=============== + +Eventlet segfaults on RedHat / Fedora +------------------------------------- + +[*If this is no longer an issue, please remove this section, thanks!*] + +On some OSes, specifically Fedora 15, the current versions of +greenlet/eventlet segfault when running keystone. To fix this, install +the development versions of greenlet and eventlet:: + + $ pip uninstall greenlet eventlet + $ cd + $ hg clone https://bitbucket.org/ambroff/greenlet + $ cd greenlet + $ sudo python setup.py install + + $ cd + $ hg clone https://bitbucket.org/which_linden/eventlet + $ cd greenlet + $ sudo python setup.py install diff --git a/docs/source/testing.rst b/docs/source/testing.rst new file mode 100644 index 0000000000..82a3360460 --- /dev/null +++ b/docs/source/testing.rst @@ -0,0 +1,77 @@ +================ +Testing Keystone +================ + +Keystone uses a number of testing methodologies to ensure correctness. + +Running Built-In Tests +====================== + +To run the full suites of tests maintained within Keystone, run:: + + $ ./run_tests.sh --with-progress + +This shows realtime feedback during test execution, and iterates over +multiple configuration variations. + +This differs from how tests are executed from the continuous integration +environment. Specifically, Jenkins doesn't care about realtime progress, +and aborts after the first test failure (a fail-fast behavior):: + + $ ./run_tests.sh + +Testing Schema Migrations +========================= + +The application of schema migrations can be tested using SQLAlchemy Migrate’s built-in test runner, one migration at a time. + +.. WARNING:: + + This may leave your database in an inconsistent state; attempt this in non-production environments only! + +This is useful for testing the *next* migration in sequence (both forward & backward) in a database under version control:: + + $ python keystone/backends/sqlalchemy/migrate_repo/manage.py test --url=sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + +This command refers to a SQLite database used for testing purposes. Depending on the migration, this command alone does not make assertions as to the integrity of your data during migration. + +Writing Tests +============= + +Tests are maintained in the ``keystone.test`` module. Unit tests are +isolated from functional tests. + +Functional Tests +---------------- + +The ``keystone.test.functional.common`` module provides a ``unittest``-based +``httplib`` client which you can extend and use for your own tests. +Generally, functional tests should serve to illustrate intended use cases +and API behaviors. To help make your tests easier to read, the test client: + +- Authenticates with a known user name and password combination +- Asserts 2xx HTTP status codes (unless told otherwise) +- Abstracts keystone REST verbs & resources into single function calls + +Testing Multiple Configurations +------------------------------- + +Several variations of the default configuration are iterated over to +ensure test coverage of mutually exclusive featuresets, such as the +various backend options. + +These configuration templates are maintained in ``keystone/test/etc`` and +are iterated over by ``run_tests.py``. + +Further Testing +=============== + +devstack_ is the *best* way to quickly deploy keystone with the rest of the +OpenStack universe and should be critical step in your development workflow! + +You may also be interested in either the `OpenStack Continuous Integration Project`_ +or the `OpenStack Integration Testing Project`_. + +.. _devstack: http://devstack.org/ +.. _OpenStack Continuous Integration Project: https://github.com/openstack/openstack-ci +.. _OpenStack Integration Testing Project: https://github.com/openstack/openstack-integration-tests From 2a91b1c06e8aa50ea603fd1c104ddea8e1649230 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 18:13:05 -0800 Subject: [PATCH 222/334] users with correct credentials but disabled are forbidden not unauthorized --- tests/test_keystoneclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index c56416eaf0..e5337f6e2b 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -214,7 +214,7 @@ class KcMasterTestCase(CompatTestCase): self.assertFalse(user.enabled) # TODO(ja): test that you can't login - self.assertRaises(client_exceptions.Unauthorized, + self.assertRaises(client_exceptions.Forbidden, self._client, username=test_username, password='password') From c83bcb1aac9bcbf0ceea7bf4defa905490bebf05 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 18:37:30 -0800 Subject: [PATCH 223/334] add checks for no password attribute --- tests/test_keystoneclient.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 1efe550b42..296b5dd8c6 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -214,6 +214,16 @@ class KcMasterTestCase(CompatTestCase): client = self.get_client() users = client.users.list() self.assertTrue(len(users) > 0) + user = users[0] + with self.assertRaises(AttributeError): + user.password + + def test_user_get(self): + client = self.get_client() + user = client.users.get(self.user_foo['id']) + with self.assertRaises(AttributeError): + user.password + def test_role_get(self): client = self.get_client() From ea78b2e4fcdefd13e35cb1eb28559f18a9cc6f1c Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 19 Jan 2012 15:54:14 -0800 Subject: [PATCH 224/334] some cli improvements prints available commands and config values when no arguments are given prints available subcommands when command is given but no subocmmand is --- bin/keystone-manage | 75 ++++++++++++++++++++++++++++++++++++++++----- keystone/config.py | 45 +++++++++++++++++++-------- 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index d290c8cdfb..ae83c1e71d 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -2,6 +2,7 @@ import os import sys +import textwrap import cli.app import cli.log @@ -22,16 +23,24 @@ from keystone import config from keystone.common import utils -CONF = config.CONF +CONF = config.Config(usage='%prog COMMAND [key1=value1 key2=value2 ...]') config.register_cli_str('endpoint', default='http://localhost:$admin_port/v2.0', - group='ks') + #group='ks', + conf=CONF) config.register_cli_str('token', default='$admin_token', - group='ks') + #group='ks', + help='asdasd', + conf=CONF) config.register_cli_bool('id-only', default=False, - group='ks') + #group='ks', + conf=CONF) +config.register_cli_str('admin-port', + conf=CONF) +config.register_cli_str('admin-token', + conf=CONF) class BaseApp(cli.log.LoggingApp): @@ -59,6 +68,8 @@ class BaseApp(cli.log.LoggingApp): class DbSync(BaseApp): + """Sync the database.""" + name = 'db_sync' def __init__(self, *args, **kw): @@ -77,12 +88,13 @@ class ClientCommand(BaseApp): def __init__(self, *args, **kw): super(ClientCommand, self).__init__(*args, **kw) if not self.ACTION_MAP: - self.ACTION_MAP = {} - self.add_param('action') + self.ACTION_MAP = {'help': 'help'} + self.add_param('action', nargs='?', default='help') self.add_param('keyvalues', nargs='*') - self.client = kc.Client(CONF.ks.endpoint, token=CONF.ks.token) + self.client = kc.Client(CONF.endpoint, token=CONF.token) self.handle = getattr(self.client, '%ss' % self.__class__.__name__.lower()) self._build_action_map() + self.usage = "foo" def _build_action_map(self): actions = {} @@ -94,6 +106,10 @@ class ClientCommand(BaseApp): def main(self): """Given some keyvalues create the appropriate data in Keystone.""" action_name = self.ACTION_MAP[self.params.action] + if action_name == 'help': + self.print_help() + sys.exit(1) + kv = self._parse_keyvalues(self.params.keyvalues) resp = getattr(self.handle, action_name)(**kv) if CONF.ks.id_only and getattr(resp, 'id'): @@ -101,24 +117,48 @@ class ClientCommand(BaseApp): return print resp + def print_help(self): + CONF.set_usage(CONF.usage.replace( + 'COMMAND', '%s SUBCOMMAND' % self.__class__.__name__.lower())) + CONF.print_help() + + methods = self._get_methods() + print_commands(methods) + + def _get_methods(self): + o = {} + for k in dir(self.handle): + if k.startswith('_'): + continue + if k in ('find', 'findall', 'api', 'resource_class'): + continue + o[k] = getattr(self.handle, k) + return o + class Role(ClientCommand): + """Role CRUD functions.""" pass class Service(ClientCommand): + """Service CRUD functions.""" pass class Token(ClientCommand): + """Token CRUD functions.""" pass class Tenant(ClientCommand): + """Tenant CRUD functions.""" pass class User(ClientCommand): + """User CRUD functions.""" + pass @@ -131,6 +171,21 @@ CMDS = {'db_sync': DbSync, } +def print_commands(cmds): + print + print "Available commands:" + o = [] + max_length = max([len(k) for k in cmds]) + 2 + for k, cmd in sorted(cmds.iteritems()): + initial_indent = '%s%s: ' % (' ' * (max_length - len(k)), k) + tw = textwrap.TextWrapper(initial_indent=initial_indent, + subsequent_indent=' ' * (max_length + 2), + width=80) + o.extend(tw.wrap( + (cmd.__doc__ and cmd.__doc__ or 'no docs').strip().split('\n')[0])) + print '\n'.join(o) + + def main(argv=None): if argv is None: argv = sys.argv @@ -141,8 +196,12 @@ def main(argv=None): config_files = None if os.path.exists(dev_conf): config_files = [dev_conf] - args = CONF(config_files=config_files, args=argv) + if len(args) < 2: + CONF.print_help() + print_commands(CMDS) + sys.exit(1) + cmd = args[1] if cmd in CMDS: CMDS[cmd](argv=(args[:1] + args[2:])).run() diff --git a/keystone/config.py b/keystone/config.py index a89c6e9031..11db8b53aa 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -11,11 +11,11 @@ from keystone.common import cfg gettext.install('keystone', unicode=1) -class Config(cfg.CommonConfigOpts): +class ConfigMixin(object): def __call__(self, config_files=None, *args, **kw): if config_files is not None: self._opts['config_file']['opt'].default = config_files - return super(Config, self).__call__(*args, **kw) + return super(ConfigMixin, self).__call__(*args, **kw) def __getitem__(self, key, default=None): return getattr(self, key, default) @@ -27,6 +27,21 @@ class Config(cfg.CommonConfigOpts): for k in self._opts: yield (k, getattr(self, k)) + def print_help(self): + self._oparser.print_help() + + def set_usage(self, usage): + self.usage = usage + self._oparser.usage = usage + + +class Config(ConfigMixin, cfg.ConfigOpts): + pass + + +class CommonConfig(ConfigMixin, cfg.CommonConfigOpts): + pass + def setup_logging(conf): """ @@ -76,33 +91,37 @@ def setup_logging(conf): def register_str(*args, **kw): - group = _ensure_group(kw) - return CONF.register_opt(cfg.StrOpt(*args, **kw), group=group) + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_opt(cfg.StrOpt(*args, **kw), group=group) def register_cli_str(*args, **kw): - group = _ensure_group(kw) - return CONF.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_cli_opt(cfg.StrOpt(*args, **kw), group=group) def register_bool(*args, **kw): - group = _ensure_group(kw) - return CONF.register_opt(cfg.BoolOpt(*args, **kw), group=group) + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_opt(cfg.BoolOpt(*args, **kw), group=group) def register_cli_bool(*args, **kw): - group = _ensure_group(kw) - return CONF.register_cli_opt(cfg.BoolOpt(*args, **kw), group=group) + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_cli_opt(cfg.BoolOpt(*args, **kw), group=group) -def _ensure_group(kw): +def _ensure_group(kw, conf): group = kw.pop('group', None) if group: - CONF.register_group(cfg.OptGroup(name=group)) + conf.register_group(cfg.OptGroup(name=group)) return group -CONF = Config(project='keystone') +CONF = CommonConfig(project='keystone') register_str('admin_token', default='ADMIN') From d8ddc074f026a2b7c72cdb2107ed8ff790a4bb5f Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 19 Jan 2012 20:36:06 -0800 Subject: [PATCH 225/334] get docs working --- docs/Makefile | 2 +- docs/generate_autodoc_index.py | 76 ---------------------------------- keystone/catalog/core.py | 3 -- keystone/contrib/ec2/core.py | 3 -- keystone/identity/core.py | 3 -- keystone/policy/core.py | 3 -- keystone/token/core.py | 3 -- 7 files changed, 1 insertion(+), 92 deletions(-) delete mode 100755 docs/generate_autodoc_index.py diff --git a/docs/Makefile b/docs/Makefile index 03e3ef3bc3..79861705e5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -47,7 +47,7 @@ clean: autodoc: $(SPHINXAPIDOC) -f -o $(SOURCEDIR) ../keystone -html: +html: autodoc $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/docs/generate_autodoc_index.py b/docs/generate_autodoc_index.py deleted file mode 100755 index 993369b028..0000000000 --- a/docs/generate_autodoc_index.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -"""Generates files for sphinx documentation using a simple Autodoc based -template. - -To use, just run as a script: - $ python doc/generate_autodoc_index.py -""" - -import os - - -base_dir = os.path.dirname(os.path.abspath(__file__)) -RSTDIR=os.path.join(base_dir, "source", "sourcecode") -SOURCEDIR=os.path.join(base_dir, "..") - -# Exclude these modules from the autodoc results -EXCLUDE_MODULES = ['keystone.backends.sqlalchemy.migrate_repo'] - -def in_exclude_list(module_name): - """Compares a module to the list of excluded modules - - Returns true if the provided module resides in or matches - an excluded module, false otherwise. - """ - for excluded_module in EXCLUDE_MODULES: - if module_name.startswith(excluded_module): - return True - return False - -def find_autodoc_modules(module_name, sourcedir): - """returns a list of modules in the SOURCE directory""" - modlist = [] - os.chdir(os.path.join(sourcedir, module_name)) - for root, dirs, files in os.walk("."): - for filename in files: - if filename.endswith(".py"): - # root = ./keystone/test/unit - # filename = base.py - elements = root.split(os.path.sep) - # replace the leading "." with the module name - elements[0] = module_name - # and get the base module name - base, extension = os.path.splitext(filename) - if not (base == "__init__"): - elements.append(base) - result = (".".join(elements)) - if not in_exclude_list(result): - modlist.append(result) - return modlist - -if not(os.path.exists(RSTDIR)): - os.mkdir(RSTDIR) - -INDEXOUT = open("%s/autoindex.rst" % RSTDIR, "w") -INDEXOUT.write("Source Code Index\n") -INDEXOUT.write("=================\n") -INDEXOUT.write(".. toctree::\n") -INDEXOUT.write(" :maxdepth: 1\n") -INDEXOUT.write("\n") - -for module in find_autodoc_modules('keystone', SOURCEDIR): - generated_file = "%s/%s.rst" % (RSTDIR, module) - - INDEXOUT.write(" %s\n" % module) - FILEOUT = open(generated_file, "w") - FILEOUT.write("The :mod:`%s` Module\n" % module) - FILEOUT.write("==============================" - "==============================" - "==============================\n") - FILEOUT.write(".. automodule:: %s\n" % module) - FILEOUT.write(" :members:\n") - FILEOUT.write(" :undoc-members:\n") - FILEOUT.write(" :show-inheritance:\n") - FILEOUT.close() - -INDEXOUT.close() diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index a99c20571e..aecae36adf 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -20,9 +20,6 @@ class Manager(manager.Manager): See :mod:`keystone.manager.Manager` for more details on how this dynamically calls the backend. - See :mod:`keystone.backends.base.Catalog` for more details on the - interface provided by backends. - """ def __init__(self): diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 1332b6431c..36a78f0898 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -22,9 +22,6 @@ class Manager(manager.Manager): See :mod:`keystone.manager.Manager` for more details on how this dynamically calls the backend. - See :mod:`keystone.backends.base.Ec2` for more details on the - interface provided by backends. - """ def __init__(self): diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 9452b909ad..bc2f7916e1 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -25,9 +25,6 @@ class Manager(manager.Manager): See :mod:`keystone.manager.Manager` for more details on how this dynamically calls the backend. - See :mod:`keystone.backends.base.Identity` for more details on the - interface provided by backends. - """ def __init__(self): diff --git a/keystone/policy/core.py b/keystone/policy/core.py index 87ad743cd6..d5af09fe08 100644 --- a/keystone/policy/core.py +++ b/keystone/policy/core.py @@ -15,9 +15,6 @@ class Manager(manager.Manager): See :mod:`keystone.manager.Manager` for more details on how this dynamically calls the backend. - See :mod:`keystone.backends.base.Policy` for more details on the - interface provided by backends. - """ def __init__(self): diff --git a/keystone/token/core.py b/keystone/token/core.py index c48ca0be05..b0385f5a60 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -15,9 +15,6 @@ class Manager(manager.Manager): See :mod:`keystone.manager.Manager` for more details on how this dynamically calls the backend. - See :mod:`keystone.backends.base.Token` for more details on the - interface provided by backends. - """ def __init__(self): From 269159f67dd5772d2cac02f9e9941785e65f0561 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 19 Jan 2012 23:00:51 -0800 Subject: [PATCH 226/334] simple docstrings for ec2 crud --- keystone/contrib/ec2/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 36a78f0898..9f451b8e2c 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -127,6 +127,7 @@ class Ec2Controller(wsgi.Application): self, token_ref, roles_ref, catalog_ref) def create_credential(self, context, user_id, tenant_id): + """Create a secret/access pair for a given user/tenant.""" # TODO(termie): validate that this request is valid for given user # tenant cred_ref = {'user_id': user_id, @@ -137,18 +138,20 @@ class Ec2Controller(wsgi.Application): return {'credential': cred_ref} def get_credentials(self, context, user_id): - """List credentials for the given user_id.""" + """List all credentials for a user.""" # TODO(termie): validate that this request is valid for given user # tenant return {'credentials': self.ec2_api.list_credentials(context, user_id)} def get_credential(self, context, user_id, credential_id): + """Lookup and retreive access/secret pair by access.""" # TODO(termie): validate that this request is valid for given user # tenant return {'credential': self.ec2_api.get_credential(context, credential_id)} def delete_credential(self, context, user_id, credential_id): + """Delete a user's access/secret pair.""" # TODO(termie): validate that this request is valid for given user # tenant return self.ec2_api.delete_credential(context, credential_id) From f94397743ea0ed008181c22ec2eafd67a83217f2 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Fri, 20 Jan 2012 00:10:47 -0800 Subject: [PATCH 227/334] ec2 docs --- keystone/contrib/ec2/core.py | 82 +++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 6 deletions(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 9f451b8e2c..b7d8c5180b 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -1,6 +1,24 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -"""Main entry point into the EC2 Credentials service.""" +"""Main entry point into the EC2 Credentials service. + +This service allows the creation of access/secret credentials used for +the ec2 interop layer of OpenStack. + +A user can create as many access/secret pairs, each of which map to a +specific tenant. This is required because OpenStack supports a user +belonging to multiple tenants, whereas the signatures created on ec2-style +requests don't allow specification of which tenant the user wishs to act +upon. + +To complete the cycle, we provide a method that OpenStack services can +use to validate a signature and get a corresponding openstack token. This +token allows method calls to other services within the context the +access/secret was created. As an example, nova requests keystone to validate +the signature of a request, receives a token, and then makes a request to glance +to list images needed to perform the requested task. + +""" import uuid @@ -67,7 +85,28 @@ class Ec2Controller(wsgi.Application): def authenticate_ec2(self, context, credentials=None, ec2Credentials=None): - """Validate a signed EC2 request and provide a token.""" + """Validate a signed EC2 request and provide a token. + + Other services (such as Nova) use this **admin** call to determine + if a request they signed received is from a valid user. + + If it is a valid signature, an openstack token that maps + to the user/tenant is returned to the caller, along with + all the other details returned from a normal token validation + call. + + The returned token is useful for making calls to other + OpenStack services within the context of the request. + + :param context: standard context + :param credentials: dict of ec2 signature + :param ec2Credentials: DEPRECATED dict of ec2 signature + :returns: token: openstack token equivalent to access key along + with the corresponding service catalog and roles + """ + + # FIXME(ja): validate that a service token was used! + # NOTE(termie): backwards compat hack if not credentials and ec2Credentials: credentials = ec2Credentials @@ -127,7 +166,16 @@ class Ec2Controller(wsgi.Application): self, token_ref, roles_ref, catalog_ref) def create_credential(self, context, user_id, tenant_id): - """Create a secret/access pair for a given user/tenant.""" + """Create a secret/access pair for use with ec2 style auth. + + Generates a new set of credentials that map the the user/tenant + pair. + + :param context: standard context + :param user_id: id of user + :param tenant_id: id of tenant + :returns: credential: dict of ec2 credential + """ # TODO(termie): validate that this request is valid for given user # tenant cred_ref = {'user_id': user_id, @@ -138,20 +186,42 @@ class Ec2Controller(wsgi.Application): return {'credential': cred_ref} def get_credentials(self, context, user_id): - """List all credentials for a user.""" + """List all credentials for a user. + + :param context: standard context + :param user_id: id of user + :returns: credentials: list of ec2 credential dicts + """ + # TODO(termie): validate that this request is valid for given user # tenant return {'credentials': self.ec2_api.list_credentials(context, user_id)} def get_credential(self, context, user_id, credential_id): - """Lookup and retreive access/secret pair by access.""" + """Retreive a user's access/secret pair by the access key. + + Grab the full access/secret pair for a given access key. + + :param context: standard context + :param user_id: id of user + :param credential_id: access key for credentials + :returns: credential: dict of ec2 credential + """ # TODO(termie): validate that this request is valid for given user # tenant return {'credential': self.ec2_api.get_credential(context, credential_id)} def delete_credential(self, context, user_id, credential_id): - """Delete a user's access/secret pair.""" + """Delete a user's access/secret pair. + + Used to revoke a user's access/secret pair + + :param context: standard context + :param user_id: id of user + :param credential_id: access key for credentials + :returns: bool: success + """ # TODO(termie): validate that this request is valid for given user # tenant return self.ec2_api.delete_credential(context, credential_id) From 28760bd33be09e29b3b1490409eced74e243d88e Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Fri, 20 Jan 2012 20:51:54 +0000 Subject: [PATCH 228/334] removing the sphinx_build from setup.py, adding how to run the docs into the README --- README.rst | 18 ++++++++++++++++++ setup.py | 26 -------------------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index d01098731d..248178b079 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,24 @@ Much of the design is precipitated from the expectation that the auth backends for most deployments will actually be shims in front of existing user systems. +----------- +Development +----------- + +Building the Documentation +-------------------------- + +The documentation is all generated with Sphinx from within the docs directory. +To generate the full set of HTML documentation: + + cd docs + make autodoc + make html + make man + +the results are in the docs/build/html and docs/build/man directories +respectively. + ------------ The Services ------------ diff --git a/setup.py b/setup.py index fa418f3107..dcd4f2e025 100755 --- a/setup.py +++ b/setup.py @@ -1,30 +1,5 @@ -import os -import subprocess - from setuptools import setup, find_packages -# If Sphinx is installed on the box running setup.py, -# enable setup.py to build the documentation, otherwise, -# just ignore it -cmdclass = {} -try: - from sphinx.setup_command import BuildDoc - - class local_BuildDoc(BuildDoc): - def run(self): - base_dir = os.path.dirname(os.path.abspath(__file__)) - subprocess.Popen(["python", "generate_autodoc_index.py"], - cwd=os.path.join(base_dir, "docs")).communicate() - for builder in ['html', 'man']: - self.builder = builder - self.finalize_options() - BuildDoc.run(self) - cmdclass['build_sphinx'] = local_BuildDoc -except: - # unable to import sphinx, politely skip past... - pass - - setup(name='keystone', version='2012.1', description="Authentication service for OpenStack", @@ -35,6 +10,5 @@ setup(name='keystone', packages=find_packages(exclude=['test', 'bin']), scripts=['bin/keystone', 'bin/keystone-manage'], zip_safe=False, - cmdclass=cmdclass, install_requires=['setuptools'], ) From 86dad078f4b4b6db49e63578ee809e08d92fcfdb Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 14:30:22 -0800 Subject: [PATCH 229/334] fix user_get/user_list tests --- tests/test_keystoneclient.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 043633bf1a..e84afb88d1 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -193,7 +193,6 @@ class KcMasterTestCase(CompatTestCase): username=self.user_foo['name'], password='invalid') - def test_user_create_update_delete(self): from keystoneclient import exceptions as client_exceptions @@ -239,15 +238,12 @@ class KcMasterTestCase(CompatTestCase): users = client.users.list() self.assertTrue(len(users) > 0) user = users[0] - with self.assertRaises(AttributeError): - user.password + self.assertRaises(AttributeError, lambda: user.password) def test_user_get(self): client = self.get_client() user = client.users.get(self.user_foo['id']) - with self.assertRaises(AttributeError): - user.password - + self.assertRaises(AttributeError, lambda: user.password) def test_role_get(self): client = self.get_client() From 36a0190f08b0a614d188db75febd2380c822c122 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 14:33:49 -0800 Subject: [PATCH 230/334] strip password from kvs backend --- keystone/common/kvs.py | 5 ++++- keystone/identity/backends/kvs.py | 26 +++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/keystone/common/kvs.py b/keystone/common/kvs.py index 7f24d9ba61..795c2f70bd 100644 --- a/keystone/common/kvs.py +++ b/keystone/common/kvs.py @@ -3,7 +3,10 @@ class DictKvs(dict): def set(self, key, value): - self[key] = value + if type(value) is type({}): + self[key] = value.copy() + else: + self[key] = value[:] def delete(self, key): del self[key] diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index 19b5d220c6..ffda9348f7 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -4,6 +4,12 @@ from keystone import identity from keystone.common import kvs +def _filter_user(user_ref): + if user_ref: + user_ref.pop('password', None) + #user_ref.pop('tenants', None) + return user_ref + class Identity(kvs.Base, identity.Driver): # Public interface def authenticate(self, user_id=None, tenant_id=None, password=None): @@ -13,7 +19,7 @@ class Identity(kvs.Base, identity.Driver): in the list of tenants on the user. """ - user_ref = self.get_user(user_id) + user_ref = self._get_user(user_id) tenant_ref = None metadata_ref = None if not user_ref or user_ref.get('password') != password: @@ -26,7 +32,7 @@ class Identity(kvs.Base, identity.Driver): metadata_ref = self.get_metadata(user_id, tenant_id) else: metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) + return (_filter_user(user_ref), tenant_ref, metadata_ref) def get_tenant(self, tenant_id): tenant_ref = self.db.get('tenant-%s' % tenant_id) @@ -36,14 +42,20 @@ class Identity(kvs.Base, identity.Driver): tenant_ref = self.db.get('tenant_name-%s' % tenant_name) return tenant_ref - def get_user(self, user_id): + def _get_user(self, user_id): user_ref = self.db.get('user-%s' % user_id) return user_ref - def get_user_by_name(self, user_name): + def _get_user_by_name(self, user_name): user_ref = self.db.get('user_name-%s' % user_name) return user_ref + def get_user(self, user_id): + return _filter_user(self._get_user(user_id)) + + def get_user_by_name(self, user_name): + return _filter_user(self._get_user_by_name(user_name)) + def get_metadata(self, user_id, tenant_id): return self.db.get('metadata-%s-%s' % (tenant_id, user_id)) @@ -61,21 +73,21 @@ class Identity(kvs.Base, identity.Driver): # These should probably be part of the high-level API def add_user_to_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) + user_ref = self._get_user(user_id) tenants = set(user_ref.get('tenants', [])) tenants.add(tenant_id) user_ref['tenants'] = list(tenants) self.update_user(user_id, user_ref) def remove_user_from_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) + user_ref = self._get_user(user_id) tenants = set(user_ref.get('tenants', [])) tenants.remove(tenant_id) user_ref['tenants'] = list(tenants) self.update_user(user_id, user_ref) def get_tenants_for_user(self, user_id): - user_ref = self.get_user(user_id) + user_ref = self._get_user(user_id) return user_ref.get('tenants', []) def get_roles_for_user_and_tenant(self, user_id, tenant_id): From c59370eda09cf0ce1f71b45f1b3321e3a687ef7b Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 14:52:53 -0800 Subject: [PATCH 231/334] rely on internal _get_user for update calls --- keystone/identity/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/identity/core.py b/keystone/identity/core.py index bc2f7916e1..484faf50ea 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -337,7 +337,7 @@ class UserController(wsgi.Application): # NOTE(termie): this is really more of a patch than a put def update_user(self, context, user_id, user): self.assert_admin(context) - user_ref = self.identity_api.get_user(context, user_id) + user_ref = self.identity_api._get_user(context, user_id) del user['id'] user_ref.update(user) self.identity_api.update_user(context, user_id, user_ref) From 3cce41e2804ca93897144b84418d523d9954c660 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 14:53:37 -0800 Subject: [PATCH 232/334] raise and catch correct authenticate error --- keystone/service.py | 13 ++++++++----- tests/test_keystoneclient.py | 3 +-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/keystone/service.py b/keystone/service.py index 186f1cf763..283f04a2f6 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -149,11 +149,14 @@ class TokenController(wsgi.Application): else: tenant_id = auth.get('tenantId', None) - (user_ref, tenant_ref, metadata_ref) = \ - self.identity_api.authenticate(context=context, - user_id=user_id, - password=password, - tenant_id=tenant_id) + try: + (user_ref, tenant_ref, metadata_ref) = \ + self.identity_api.authenticate(context=context, + user_id=user_id, + password=password, + tenant_id=tenant_id) + except AssertionError as e: + raise webob.exc.HTTPForbidden(e.message) token_ref = self.token_api.create_token( context, token_id, dict(expires='', id=token_id, diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index e84afb88d1..16e8fbf8dd 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -212,8 +212,7 @@ class KcMasterTestCase(CompatTestCase): user = client.users.get(user.id) self.assertFalse(user.enabled) - # TODO(ja): test that you can't login - self.assertRaises(client_exceptions.Forbidden, + self.assertRaises(client_exceptions.AuthorizationFailure, self._client, username=test_username, password='password') From 57b24dde131a349afb7a533b6a54f6ab1362eeae Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 14:56:38 -0800 Subject: [PATCH 233/334] strip password from sql backend --- keystone/identity/backends/sql.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index b2a3ffad34..a614ebf945 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -5,6 +5,12 @@ from keystone.common import sql from keystone.common.sql import migration +def _filter_user(user_ref): + if user_ref: + user_ref.pop('password', None) + return user_ref + + class User(sql.ModelBase, sql.DictBase): __tablename__ = 'user' id = sql.Column(sql.String(64), primary_key=True) @@ -97,7 +103,7 @@ class Identity(sql.Base, identity.Driver): in the list of tenants on the user. """ - user_ref = self.get_user(user_id) + user_ref = self._get_user(user_id) tenant_ref = None metadata_ref = None if not user_ref or user_ref.get('password') != password: @@ -112,7 +118,7 @@ class Identity(sql.Base, identity.Driver): metadata_ref = self.get_metadata(user_id, tenant_id) else: metadata_ref = {} - return (user_ref, tenant_ref, metadata_ref) + return (_filter_user(user_ref), tenant_ref, metadata_ref) def get_tenant(self, tenant_id): session = self.get_session() @@ -128,20 +134,26 @@ class Identity(sql.Base, identity.Driver): return return tenant_ref.to_dict() - def get_user(self, user_id): + def _get_user(self, user_id): session = self.get_session() user_ref = session.query(User).filter_by(id=user_id).first() if not user_ref: return return user_ref.to_dict() - def get_user_by_name(self, user_name): + def _get_user_by_name(self, user_name): session = self.get_session() user_ref = session.query(User).filter_by(name=user_name).first() if not user_ref: return return user_ref.to_dict() + def get_user(self, user_id): + return _filter_user(self._get_user(user_id)) + + def get_user_by_name(self, user_name): + return _filter_user(self._get_user_by_name(user_name)) + def get_metadata(self, user_id, tenant_id): session = self.get_session() metadata_ref = session.query(Metadata)\ @@ -158,7 +170,7 @@ class Identity(sql.Base, identity.Driver): def list_users(self): session = self.get_session() user_refs = session.query(User) - return [x.to_dict() for x in user_refs] + return [_filter_user(x.to_dict()) for x in user_refs] def list_roles(self): session = self.get_session() From 2ebb89bf8024d2700a9c884763da795fba35b14f Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 16:35:41 -0800 Subject: [PATCH 234/334] fix invalid_password, skip ec2 tests --- tests/test_keystoneclient.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 16e8fbf8dd..32a8cbc26e 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -1,4 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import nose.exc + from keystone import config from keystone import test @@ -188,7 +190,7 @@ class KcMasterTestCase(CompatTestCase): password=self.user_foo['password']) good_client.tenants.list() - self.assertRaises(client_exceptions.Unauthorized, + self.assertRaises(client_exceptions.AuthorizationFailure, self._client, username=self.user_foo['name'], password='invalid') @@ -295,6 +297,7 @@ class KcMasterTestCase(CompatTestCase): self.assertEquals(creds, []) def test_ec2_credentials_list_unauthorized_user(self): + raise nose.exc.SkipTest('TODO') from keystoneclient import exceptions as client_exceptions two = self.get_client(self.user_two) @@ -302,6 +305,7 @@ class KcMasterTestCase(CompatTestCase): self.user_foo['id']) def test_ec2_credentials_get_unauthorized_user(self): + raise nose.exc.SkipTest('TODO') from keystoneclient import exceptions as client_exceptions foo = self.get_client() @@ -314,6 +318,7 @@ class KcMasterTestCase(CompatTestCase): foo.ec2.delete(self.user_foo['id'], cred.access) def test_ec2_credentials_delete_unauthorized_user(self): + raise nose.exc.SkipTest('TODO') from keystoneclient import exceptions as client_exceptions foo = self.get_client() From 5a8a8ae6d7341138ff28395d67d4acc8e467e7ba Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 16:35:58 -0800 Subject: [PATCH 235/334] turn off echo --- keystone/common/sql/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index 9c51d782bf..172a2a6653 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -103,7 +103,7 @@ class Base(object): connection_dict = sqlalchemy.engine.url.make_url(CONF.sql.connection) engine_args = {"pool_recycle": CONF.sql.idle_timeout, - "echo": True, + "echo": False, } if "sqlite" in connection_dict.drivername: From 8ffee09635c5be69a578376d309bcfc5b7b8c927 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 16:49:28 -0800 Subject: [PATCH 236/334] don't allow disabled users to authenticate --- keystone/service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/keystone/service.py b/keystone/service.py index 283f04a2f6..b017d69768 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -155,8 +155,13 @@ class TokenController(wsgi.Application): user_id=user_id, password=password, tenant_id=tenant_id) + + # If the user is disabled don't allow them to authenticate + if not user_ref.get('enabled', True): + raise webob.exc.HTTPForbidden('User has been disabled') except AssertionError as e: raise webob.exc.HTTPForbidden(e.message) + token_ref = self.token_api.create_token( context, token_id, dict(expires='', id=token_id, From 8ad8d88dc65daf2662fde34033619a4918be6129 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:18:30 -0800 Subject: [PATCH 237/334] flip actual and expected to match common api --- keystone/test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/keystone/test.py b/keystone/test.py index 04502f80b3..82d2c77df4 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -192,17 +192,17 @@ class TestCase(unittest.TestCase): sys.path.insert(0, path) self._paths.append(path) - def assertListEquals(self, expected, actual): + def assertListEquals(self, actual, expected): copy = expected[:] #print expected, actual - self.assertEquals(len(expected), len(actual)) + self.assertEquals(len(actual), len(expected)) while copy: item = copy.pop() matched = False for x in actual: #print 'COMPARE', item, x, try: - self.assertDeepEquals(item, x) + self.assertDeepEquals(x, item) matched = True #print 'MATCHED' break @@ -213,7 +213,7 @@ class TestCase(unittest.TestCase): raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) - def assertDictEquals(self, expected, actual): + def assertDictEquals(self, actual, expected): for k in expected: self.assertTrue(k in actual, "Expected key %s not in %s." % (k, actual)) @@ -223,15 +223,15 @@ class TestCase(unittest.TestCase): self.assertTrue(k in expected, "Unexpected key %s in %s." % (k, actual)) - def assertDeepEquals(self, expected, actual): + def assertDeepEquals(self, actual, expected): try: if type(expected) is type([]) or type(expected) is type(tuple()): # assert items equal, ignore order - self.assertListEquals(expected, actual) + self.assertListEquals(actual, expected) elif type(expected) is type({}): - self.assertDictEquals(expected, actual) + self.assertDictEquals(actual, expected) else: - self.assertEquals(expected, actual) + self.assertEquals(actual, expected) except AssertionError as e: raise raise AssertionError('Expected: %s\n Got: %s' % (expected, actual)) From da4f95527953938692627a9ba12f21731cb1c12e Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:18:47 -0800 Subject: [PATCH 238/334] strip password before checking output --- tests/test_backend.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_backend.py b/tests/test_backend.py index 1d4edd5b76..30ff3edad3 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -24,6 +24,10 @@ class IdentityTests(object): user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( user_id=self.user_foo['id'], password=self.user_foo['password']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') self.assertDictEquals(user_ref, self.user_foo) self.assert_(tenant_ref is None) self.assert_(not metadata_ref) @@ -33,6 +37,10 @@ class IdentityTests(object): user_id=self.user_foo['id'], tenant_id=self.tenant_bar['id'], password=self.user_foo['password']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') self.assertDictEquals(user_ref, self.user_foo) self.assertDictEquals(tenant_ref, self.tenant_bar) self.assertDictEquals(metadata_ref, self.metadata_foobar) @@ -63,6 +71,10 @@ class IdentityTests(object): def test_get_user(self): user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') self.assertDictEquals(user_ref, self.user_foo) def test_get_metadata_bad_user(self): From e4a00e05e93094f7b7f9ff8dbac374c56669ffc6 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:19:01 -0800 Subject: [PATCH 239/334] fix some more pass-by-reference bugs --- keystone/identity/backends/kvs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index ffda9348f7..687aba3157 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -6,8 +6,9 @@ from keystone.common import kvs def _filter_user(user_ref): if user_ref: + user_ref = user_ref.copy() user_ref.pop('password', None) - #user_ref.pop('tenants', None) + user_ref.pop('tenants', None) return user_ref class Identity(kvs.Base, identity.Driver): From aaf75e9bb0f8a4d19ffb1bac39a737982ca8e6bf Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:22:10 -0800 Subject: [PATCH 240/334] fix pep8 --- keystone/contrib/ec2/core.py | 8 ++++---- keystone/identity/backends/kvs.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index b7d8c5180b..fab5f3891b 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -3,7 +3,7 @@ """Main entry point into the EC2 Credentials service. This service allows the creation of access/secret credentials used for -the ec2 interop layer of OpenStack. +the ec2 interop layer of OpenStack. A user can create as many access/secret pairs, each of which map to a specific tenant. This is required because OpenStack supports a user @@ -15,8 +15,8 @@ To complete the cycle, we provide a method that OpenStack services can use to validate a signature and get a corresponding openstack token. This token allows method calls to other services within the context the access/secret was created. As an example, nova requests keystone to validate -the signature of a request, receives a token, and then makes a request to glance -to list images needed to perform the requested task. +the signature of a request, receives a token, and then makes a request to +glance to list images needed to perform the requested task. """ @@ -95,7 +95,7 @@ class Ec2Controller(wsgi.Application): all the other details returned from a normal token validation call. - The returned token is useful for making calls to other + The returned token is useful for making calls to other OpenStack services within the context of the request. :param context: standard context diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index 687aba3157..26209355e9 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -11,6 +11,7 @@ def _filter_user(user_ref): user_ref.pop('tenants', None) return user_ref + class Identity(kvs.Base, identity.Driver): # Public interface def authenticate(self, user_id=None, tenant_id=None, password=None): From 9f0bb492128e396dcdb0d0be76bce913718a6611 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:27:36 -0800 Subject: [PATCH 241/334] some quick fixes to cli, tests incoming --- bin/keystone-manage | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index ae83c1e71d..4bcd5d7800 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -23,7 +23,8 @@ from keystone import config from keystone.common import utils -CONF = config.Config(usage='%prog COMMAND [key1=value1 key2=value2 ...]') +CONF = config.CONF +CONF.set_usage('%prog COMMAND [key1=value1 key2=value2 ...]') config.register_cli_str('endpoint', default='http://localhost:$admin_port/v2.0', #group='ks', @@ -37,10 +38,6 @@ config.register_cli_bool('id-only', default=False, #group='ks', conf=CONF) -config.register_cli_str('admin-port', - conf=CONF) -config.register_cli_str('admin-token', - conf=CONF) class BaseApp(cli.log.LoggingApp): @@ -112,7 +109,7 @@ class ClientCommand(BaseApp): kv = self._parse_keyvalues(self.params.keyvalues) resp = getattr(self.handle, action_name)(**kv) - if CONF.ks.id_only and getattr(resp, 'id'): + if CONF.id_only and getattr(resp, 'id'): print resp.id return print resp From e344821225d0f44461eff9376de8509660ecd4e1 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 23 Jan 2012 17:47:48 -0800 Subject: [PATCH 242/334] fix token vs auth_token --- bin/keystone-manage | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index 4bcd5d7800..ff97f7e4d4 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -1,5 +1,6 @@ #!/usr/bin/env python +import logging import os import sys import textwrap @@ -29,7 +30,7 @@ config.register_cli_str('endpoint', default='http://localhost:$admin_port/v2.0', #group='ks', conf=CONF) -config.register_cli_str('token', +config.register_cli_str('auth_token', default='$admin_token', #group='ks', help='asdasd', @@ -88,10 +89,9 @@ class ClientCommand(BaseApp): self.ACTION_MAP = {'help': 'help'} self.add_param('action', nargs='?', default='help') self.add_param('keyvalues', nargs='*') - self.client = kc.Client(CONF.endpoint, token=CONF.token) + self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) self.handle = getattr(self.client, '%ss' % self.__class__.__name__.lower()) self._build_action_map() - self.usage = "foo" def _build_action_map(self): actions = {} @@ -108,7 +108,13 @@ class ClientCommand(BaseApp): sys.exit(1) kv = self._parse_keyvalues(self.params.keyvalues) - resp = getattr(self.handle, action_name)(**kv) + try: + f = getattr(self.handle, action_name) + resp = f(**kv) + except Exception: + logging.exception('') + raise + if CONF.id_only and getattr(resp, 'id'): print resp.id return From 4899210334a030c541e13aa994b25b3996bb3cc9 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 00:56:53 -0800 Subject: [PATCH 243/334] bcrypt the passwords --- keystone/common/utils.py | 25 +++++++++++++++++++++++++ keystone/config.py | 12 ++++++++++++ keystone/identity/backends/kvs.py | 27 +++++++++++++++++++-------- keystone/identity/backends/sql.py | 13 ++++++++++++- keystone/identity/core.py | 7 ++----- keystone/test.py | 10 ++++++++-- tests/test_backend.py | 5 +++++ tests/test_keystoneclient.py | 10 ++-------- tests/test_keystoneclient_sql.py | 2 +- tests/test_overrides.conf | 3 +++ tools/pip-requires | 1 + 11 files changed, 90 insertions(+), 25 deletions(-) diff --git a/keystone/common/utils.py b/keystone/common/utils.py index c3f19037b1..3ee9489441 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -25,9 +25,16 @@ import subprocess import sys import urllib +import bcrypt + +from keystone import config from keystone.common import logging +CONF = config.CONF +config.register_int('bcrypt_strength', default=12) + + def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') @@ -134,6 +141,24 @@ class Ec2Signer(object): return b64 +def hash_password(password): + """Hash a password. Hard.""" + salt = bcrypt.gensalt(CONF.bcrypt_strength) + return bcrypt.hashpw(password, salt) + + +def check_password(password, hashed): + """Check that a plaintext password matches hashed. + + Due to the way bcrypt works, hashing a password with the hashed + version of that password as salt will return the hashed version + of that password (mostly). Neat! + + """ + check = bcrypt.hashpw(password, hashed[:29]) + return check == hashed + + # From python 2.7 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. diff --git a/keystone/config.py b/keystone/config.py index 11db8b53aa..2d69d7a644 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -114,6 +114,18 @@ def register_cli_bool(*args, **kw): return conf.register_cli_opt(cfg.BoolOpt(*args, **kw), group=group) +def register_int(*args, **kw): + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_opt(cfg.IntOpt(*args, **kw), group=group) + + +def register_cli_int(*args, **kw): + conf = kw.pop('conf', CONF) + group = _ensure_group(kw, conf) + return conf.register_cli_opt(cfg.IntOpt(*args, **kw), group=group) + + def _ensure_group(kw, conf): group = kw.pop('group', None) if group: diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index 26209355e9..8215054b49 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -2,6 +2,7 @@ from keystone import identity from keystone.common import kvs +from keystone.common import utils def _filter_user(user_ref): @@ -12,6 +13,13 @@ def _filter_user(user_ref): return user_ref +def _ensure_hashed_password(user_ref): + pw = user_ref.get('password', None) + if pw is not None: + user_ref['password'] = utils.hash_password(pw) + return user_ref + + class Identity(kvs.Base, identity.Driver): # Public interface def authenticate(self, user_id=None, tenant_id=None, password=None): @@ -24,7 +32,8 @@ class Identity(kvs.Base, identity.Driver): user_ref = self._get_user(user_id) tenant_ref = None metadata_ref = None - if not user_ref or user_ref.get('password') != password: + if (not user_ref + or not utils.check_password(password, user_ref.get('password'))): raise AssertionError('Invalid user / password') if tenant_id and tenant_id not in user_ref['tenants']: raise AssertionError('Invalid tenant') @@ -78,15 +87,13 @@ class Identity(kvs.Base, identity.Driver): user_ref = self._get_user(user_id) tenants = set(user_ref.get('tenants', [])) tenants.add(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + self.update_user(user_id, {'tenants': list(tenants)}) def remove_user_from_tenant(self, tenant_id, user_id): user_ref = self._get_user(user_id) tenants = set(user_ref.get('tenants', [])) tenants.remove(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) + self.update_user(user_id, {'tenants': list(tenants)}) def get_tenants_for_user(self, user_id): user_ref = self._get_user(user_id) @@ -118,6 +125,7 @@ class Identity(kvs.Base, identity.Driver): # CRUD def create_user(self, user_id, user): + user = _ensure_hashed_password(user) self.db.set('user-%s' % user_id, user) self.db.set('user_name-%s' % user['name'], user) user_list = set(self.db.get('user_list', [])) @@ -128,10 +136,13 @@ class Identity(kvs.Base, identity.Driver): def update_user(self, user_id, user): # get the old name and delete it too old_user = self.db.get('user-%s' % user_id) + new_user = old_user.copy() + user = _ensure_hashed_password(user) + new_user.update(user) self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % user_id, user) - self.db.set('user_name-%s' % user['name'], user) - return user + self.db.set('user-%s' % user_id, new_user) + self.db.set('user_name-%s' % new_user['name'], new_user) + return new_user def delete_user(self, user_id): old_user = self.db.get('user-%s' % user_id) diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index a614ebf945..8a3db42248 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -2,6 +2,7 @@ from keystone import identity from keystone.common import sql +from keystone.common import utils from keystone.common.sql import migration @@ -11,6 +12,13 @@ def _filter_user(user_ref): return user_ref +def _ensure_hashed_password(user_ref): + pw = user_ref.get('password', None) + if pw is not None: + user_ref['password'] = utils.hash_password(pw) + return user_ref + + class User(sql.ModelBase, sql.DictBase): __tablename__ = 'user' id = sql.Column(sql.String(64), primary_key=True) @@ -106,7 +114,8 @@ class Identity(sql.Base, identity.Driver): user_ref = self._get_user(user_id) tenant_ref = None metadata_ref = None - if not user_ref or user_ref.get('password') != password: + if (not user_ref + or not utils.check_password(password, user_ref.get('password'))): raise AssertionError('Invalid user / password') tenants = self.get_tenants_for_user(user_id) @@ -246,6 +255,7 @@ class Identity(sql.Base, identity.Driver): # CRUD def create_user(self, user_id, user): + user = _ensure_hashed_password(user) session = self.get_session() with session.begin(): user_ref = User.from_dict(user) @@ -258,6 +268,7 @@ class Identity(sql.Base, identity.Driver): with session.begin(): user_ref = session.query(User).filter_by(id=user_id).first() old_user_dict = user_ref.to_dict() + user = _ensure_hashed_password(user) for k in user: old_user_dict[k] = user[k] new_user = User.from_dict(old_user_dict) diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 484faf50ea..e841898545 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -334,13 +334,10 @@ class UserController(wsgi.Application): self.identity_api.add_user_to_tenant(tenant_id, user_id) return {'user': new_user_ref} - # NOTE(termie): this is really more of a patch than a put def update_user(self, context, user_id, user): + # NOTE(termie): this is really more of a patch than a put self.assert_admin(context) - user_ref = self.identity_api._get_user(context, user_id) - del user['id'] - user_ref.update(user) - self.identity_api.update_user(context, user_id, user_ref) + user_ref = self.identity_api.update_user(context, user_id, user) return {'user': user_ref} def delete_user(self, context, user_id): diff --git a/keystone/test.py b/keystone/test.py index 82d2c77df4..9e1137a7a5 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -107,6 +107,11 @@ class TestCase(unittest.TestCase): def setUp(self): super(TestCase, self).setUp() + self.config() + + def config(self): + CONF(config_files=[etcdir('keystone.conf'), + testsdir('test_overrides.conf')]) def tearDown(self): for path in self._paths: @@ -137,10 +142,11 @@ class TestCase(unittest.TestCase): for user in fixtures.USERS: user_copy = user.copy() tenants = user_copy.pop('tenants') - rv = self.identity_api.create_user(user['id'], user_copy) + rv = self.identity_api.create_user(user['id'], user_copy.copy()) for tenant_id in tenants: self.identity_api.add_user_to_tenant(tenant_id, user['id']) - setattr(self, 'user_%s' % user['id'], rv) + setattr(self, 'user_%s' % user['id'], user_copy) + print user_copy for role in fixtures.ROLES: rv = self.identity_api.create_role(role['id'], role) diff --git a/tests/test_backend.py b/tests/test_backend.py index 30ff3edad3..151700aa23 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -45,6 +45,11 @@ class IdentityTests(object): self.assertDictEquals(tenant_ref, self.tenant_bar) self.assertDictEquals(metadata_ref, self.metadata_foobar) + def test_password_hashed(self): + user_ref = self.identity_api._get_user(self.user_foo['id']) + self.assertNotEqual(user_ref['password'], self.user_foo['password']) + + def test_get_tenant_bad_tenant(self): tenant_ref = self.identity_api.get_tenant( tenant_id=self.tenant_bar['id'] + 'WRONG') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 32a8cbc26e..d38313d303 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -45,7 +45,6 @@ class KcMasterTestCase(CompatTestCase): from keystoneclient.v2_0 import client as ks_client reload(ks_client) - self._config() self.public_app = self.loadapp('keystone', name='main') self.admin_app = self.loadapp('keystone', name='admin') @@ -63,10 +62,6 @@ class KcMasterTestCase(CompatTestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) - def _config(self): - CONF(config_files=[test.etcdir('keystone.conf'), - test.testsdir('test_overrides.conf')]) - def get_client(self, user_ref=None, tenant_ref=None): if user_ref is None: user_ref = self.user_foo @@ -87,10 +82,9 @@ class KcMasterTestCase(CompatTestCase): self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_tenant_id_and_tenants(self): - client = self._client(username='FOO', - password='foo2', + client = self._client(username=self.user_foo['name'], + password=self.user_foo['password'], tenant_id='bar') - tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py index ce8ce01151..e365f83c5e 100644 --- a/tests/test_keystoneclient_sql.py +++ b/tests/test_keystoneclient_sql.py @@ -11,7 +11,7 @@ CONF = config.CONF class KcMasterSqlTestCase(test_keystoneclient.KcMasterTestCase): - def _config(self): + def config(self): CONF(config_files=[test.etcdir('keystone.conf'), test.testsdir('test_overrides.conf'), test.testsdir('backend_sql.conf')]) diff --git a/tests/test_overrides.conf b/tests/test_overrides.conf index fdeb6e4b2d..399110d7e8 100644 --- a/tests/test_overrides.conf +++ b/tests/test_overrides.conf @@ -1,2 +1,5 @@ +[DEFAULT] +bcrypt_strength = 2 + [catalog] template_file = default_catalog.templates diff --git a/tools/pip-requires b/tools/pip-requires index 062e94ee5d..d4157f8ef1 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -7,5 +7,6 @@ routes pycli sqlalchemy sqlalchemy-migrate +py-bcrypt -e git+https://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient From 51a2c185ff41b373def1f6c7f7635f652c0d1dff Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 15:55:44 -0800 Subject: [PATCH 244/334] fix middleware --- keystone/middleware/auth_token.py | 2 +- keystone/middleware/swift_auth.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/keystone/middleware/auth_token.py b/keystone/middleware/auth_token.py index c4e28589d0..d806dd01f7 100755 --- a/keystone/middleware/auth_token.py +++ b/keystone/middleware/auth_token.py @@ -76,7 +76,7 @@ import webob import webob.exc from webob.exc import HTTPUnauthorized -from keystone.bufferedhttp import http_connect_raw as http_connect +from keystone.common.bufferedhttp import http_connect_raw as http_connect PROTOCOL_NAME = "Token Authentication" diff --git a/keystone/middleware/swift_auth.py b/keystone/middleware/swift_auth.py index 75120c340e..35bab6a337 100755 --- a/keystone/middleware/swift_auth.py +++ b/keystone/middleware/swift_auth.py @@ -43,7 +43,7 @@ import json from urlparse import urlparse from webob.exc import HTTPUnauthorized, HTTPNotFound, HTTPExpectationFailed -from keystone.bufferedhttp import http_connect_raw as http_connect +from keystone.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed from swift.common.utils import get_logger, split_path From f5dbc98dffa3216506453c6dbf038972577fe3fc Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 16:33:31 -0800 Subject: [PATCH 245/334] add ec2 credentials to the cli --- bin/keystone-manage | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/keystone-manage b/bin/keystone-manage index ff97f7e4d4..9729be09e2 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -165,12 +165,17 @@ class User(ClientCommand): pass +class Ec2(ClientCommand): + pass + + CMDS = {'db_sync': DbSync, 'role': Role, 'service': Service, 'token': Token, 'tenant': Tenant, 'user': User, + 'ec2': Ec2, } From a6a6124ccd5790bd60eef55880403a4a9f736564 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 16:49:29 -0800 Subject: [PATCH 246/334] allow class names to be different from attr names --- bin/keystone-manage | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index 9729be09e2..5193a43a05 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -83,6 +83,12 @@ class DbSync(BaseApp): class ClientCommand(BaseApp): ACTION_MAP = None + def _attr_name(self): + return '%ss' % self.__class__.__name__.lower() + + def _cmd_name(self): + return self.__class__.__name__.lower() + def __init__(self, *args, **kw): super(ClientCommand, self).__init__(*args, **kw) if not self.ACTION_MAP: @@ -90,7 +96,7 @@ class ClientCommand(BaseApp): self.add_param('action', nargs='?', default='help') self.add_param('keyvalues', nargs='*') self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) - self.handle = getattr(self.client, '%ss' % self.__class__.__name__.lower()) + self.handle = getattr(self.client, self._attr_name()) self._build_action_map() def _build_action_map(self): @@ -122,7 +128,7 @@ class ClientCommand(BaseApp): def print_help(self): CONF.set_usage(CONF.usage.replace( - 'COMMAND', '%s SUBCOMMAND' % self.__class__.__name__.lower())) + 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) CONF.print_help() methods = self._get_methods() @@ -166,7 +172,8 @@ class User(ClientCommand): class Ec2(ClientCommand): - pass + def _attr_name(self): + return self.__class__.__name__.lower() CMDS = {'db_sync': DbSync, From de6a98a1381090583f253ccbaa9c5af3c392e877 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 20:28:18 -0800 Subject: [PATCH 247/334] move cli code into a module for testing --- bin/keystone-manage | 204 +------------------------------------------ keystone/cli.py | 206 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 201 deletions(-) create mode 100644 keystone/cli.py diff --git a/bin/keystone-manage b/bin/keystone-manage index 5193a43a05..bcc27195eb 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -1,13 +1,7 @@ #!/usr/bin/env python -import logging import os import sys -import textwrap - -import cli.app -import cli.log -from keystoneclient.v2_0 import client as kc # If ../../keystone/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -20,207 +14,15 @@ if os.path.exists(os.path.join(possible_topdir, sys.path.insert(0, possible_topdir) -from keystone import config -from keystone.common import utils +from keystone import cli -CONF = config.CONF -CONF.set_usage('%prog COMMAND [key1=value1 key2=value2 ...]') -config.register_cli_str('endpoint', - default='http://localhost:$admin_port/v2.0', - #group='ks', - conf=CONF) -config.register_cli_str('auth_token', - default='$admin_token', - #group='ks', - help='asdasd', - conf=CONF) -config.register_cli_bool('id-only', - default=False, - #group='ks', - conf=CONF) - - -class BaseApp(cli.log.LoggingApp): - def __init__(self, *args, **kw): - kw.setdefault('name', self.__class__.__name__.lower()) - super(BaseApp, self).__init__(*args, **kw) - - def add_default_params(self): - for args, kw in DEFAULT_PARAMS: - self.add_param(*args, **kw) - - def _parse_keyvalues(self, args): - kv = {} - for x in args: - key, value = x.split('=', 1) - # make lists if there are multiple values - if key.endswith('[]'): - key = key[:-2] - existing = kv.get(key, []) - existing.append(value) - kv[key] = existing - else: - kv[key] = value - return kv - - -class DbSync(BaseApp): - """Sync the database.""" - - name = 'db_sync' - - def __init__(self, *args, **kw): - super(DbSync, self).__init__(*args, **kw) - - def main(self): - for k in ['identity', 'catalog', 'policy', 'token']: - driver = utils.import_object(getattr(CONF, k).driver) - if hasattr(driver, 'db_sync'): - driver.db_sync() - - -class ClientCommand(BaseApp): - ACTION_MAP = None - - def _attr_name(self): - return '%ss' % self.__class__.__name__.lower() - - def _cmd_name(self): - return self.__class__.__name__.lower() - - def __init__(self, *args, **kw): - super(ClientCommand, self).__init__(*args, **kw) - if not self.ACTION_MAP: - self.ACTION_MAP = {'help': 'help'} - self.add_param('action', nargs='?', default='help') - self.add_param('keyvalues', nargs='*') - self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) - self.handle = getattr(self.client, self._attr_name()) - self._build_action_map() - - def _build_action_map(self): - actions = {} - for k in dir(self.handle): - if not k.startswith('_'): - actions[k] = k - self.ACTION_MAP.update(actions) - - def main(self): - """Given some keyvalues create the appropriate data in Keystone.""" - action_name = self.ACTION_MAP[self.params.action] - if action_name == 'help': - self.print_help() - sys.exit(1) - - kv = self._parse_keyvalues(self.params.keyvalues) - try: - f = getattr(self.handle, action_name) - resp = f(**kv) - except Exception: - logging.exception('') - raise - - if CONF.id_only and getattr(resp, 'id'): - print resp.id - return - print resp - - def print_help(self): - CONF.set_usage(CONF.usage.replace( - 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) - CONF.print_help() - - methods = self._get_methods() - print_commands(methods) - - def _get_methods(self): - o = {} - for k in dir(self.handle): - if k.startswith('_'): - continue - if k in ('find', 'findall', 'api', 'resource_class'): - continue - o[k] = getattr(self.handle, k) - return o - - -class Role(ClientCommand): - """Role CRUD functions.""" - pass - - -class Service(ClientCommand): - """Service CRUD functions.""" - pass - - -class Token(ClientCommand): - """Token CRUD functions.""" - pass - - -class Tenant(ClientCommand): - """Tenant CRUD functions.""" - pass - - -class User(ClientCommand): - """User CRUD functions.""" - - pass - - -class Ec2(ClientCommand): - def _attr_name(self): - return self.__class__.__name__.lower() - - -CMDS = {'db_sync': DbSync, - 'role': Role, - 'service': Service, - 'token': Token, - 'tenant': Tenant, - 'user': User, - 'ec2': Ec2, - } - - -def print_commands(cmds): - print - print "Available commands:" - o = [] - max_length = max([len(k) for k in cmds]) + 2 - for k, cmd in sorted(cmds.iteritems()): - initial_indent = '%s%s: ' % (' ' * (max_length - len(k)), k) - tw = textwrap.TextWrapper(initial_indent=initial_indent, - subsequent_indent=' ' * (max_length + 2), - width=80) - o.extend(tw.wrap( - (cmd.__doc__ and cmd.__doc__ or 'no docs').strip().split('\n')[0])) - print '\n'.join(o) - - -def main(argv=None): - if argv is None: - argv = sys.argv - +if __name__ == '__main__': dev_conf = os.path.join(possible_topdir, 'etc', 'keystone.conf') config_files = None if os.path.exists(dev_conf): config_files = [dev_conf] - args = CONF(config_files=config_files, args=argv) - if len(args) < 2: - CONF.print_help() - print_commands(CMDS) - sys.exit(1) - cmd = args[1] - if cmd in CMDS: - CMDS[cmd](argv=(args[:1] + args[2:])).run() - - -if __name__ == '__main__': - main() + cli.main(argv=sys.argv, config_files=config_files) diff --git a/keystone/cli.py b/keystone/cli.py new file mode 100644 index 0000000000..814ef51304 --- /dev/null +++ b/keystone/cli.py @@ -0,0 +1,206 @@ +from __future__ import absolute_import + +import logging +import os +import sys +import textwrap + +import cli.app +import cli.log +from keystoneclient.v2_0 import client as kc + +from keystone import config +from keystone.common import utils + + +CONF = config.CONF +CONF.set_usage('%prog COMMAND [key1=value1 key2=value2 ...]') +config.register_cli_str('endpoint', + default='http://localhost:$admin_port/v2.0', + #group='ks', + conf=CONF) +config.register_cli_str('auth_token', + default='$admin_token', + #group='ks', + help='asdasd', + conf=CONF) +config.register_cli_bool('id-only', + default=False, + #group='ks', + conf=CONF) + + +class BaseApp(cli.log.LoggingApp): + def __init__(self, *args, **kw): + kw.setdefault('name', self.__class__.__name__.lower()) + super(BaseApp, self).__init__(*args, **kw) + + def add_default_params(self): + for args, kw in DEFAULT_PARAMS: + self.add_param(*args, **kw) + + def _parse_keyvalues(self, args): + kv = {} + for x in args: + key, value = x.split('=', 1) + # make lists if there are multiple values + if key.endswith('[]'): + key = key[:-2] + existing = kv.get(key, []) + existing.append(value) + kv[key] = existing + else: + kv[key] = value + return kv + + +class DbSync(BaseApp): + """Sync the database.""" + + name = 'db_sync' + + def __init__(self, *args, **kw): + super(DbSync, self).__init__(*args, **kw) + + def main(self): + for k in ['identity', 'catalog', 'policy', 'token']: + driver = utils.import_object(getattr(CONF, k).driver) + if hasattr(driver, 'db_sync'): + driver.db_sync() + + +class ClientCommand(BaseApp): + ACTION_MAP = None + + def _attr_name(self): + return '%ss' % self.__class__.__name__.lower() + + def _cmd_name(self): + return self.__class__.__name__.lower() + + def __init__(self, *args, **kw): + super(ClientCommand, self).__init__(*args, **kw) + if not self.ACTION_MAP: + self.ACTION_MAP = {'help': 'help'} + self.add_param('action', nargs='?', default='help') + self.add_param('keyvalues', nargs='*') + self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) + self.handle = getattr(self.client, self._attr_name()) + self._build_action_map() + + def _build_action_map(self): + actions = {} + for k in dir(self.handle): + if not k.startswith('_'): + actions[k] = k + self.ACTION_MAP.update(actions) + + def main(self): + """Given some keyvalues create the appropriate data in Keystone.""" + action_name = self.ACTION_MAP[self.params.action] + if action_name == 'help': + self.print_help() + sys.exit(1) + + kv = self._parse_keyvalues(self.params.keyvalues) + try: + f = getattr(self.handle, action_name) + resp = f(**kv) + except Exception: + logging.exception('') + raise + + if CONF.id_only and getattr(resp, 'id'): + print resp.id + return + print resp + + def print_help(self): + CONF.set_usage(CONF.usage.replace( + 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) + CONF.print_help() + + methods = self._get_methods() + print_commands(methods) + + def _get_methods(self): + o = {} + for k in dir(self.handle): + if k.startswith('_'): + continue + if k in ('find', 'findall', 'api', 'resource_class'): + continue + o[k] = getattr(self.handle, k) + return o + + +class Role(ClientCommand): + """Role CRUD functions.""" + pass + + +class Service(ClientCommand): + """Service CRUD functions.""" + pass + + +class Token(ClientCommand): + """Token CRUD functions.""" + pass + + +class Tenant(ClientCommand): + """Tenant CRUD functions.""" + pass + + +class User(ClientCommand): + """User CRUD functions.""" + + pass + + +class Ec2(ClientCommand): + def _attr_name(self): + return self.__class__.__name__.lower() + + +CMDS = {'db_sync': DbSync, + 'role': Role, + 'service': Service, + 'token': Token, + 'tenant': Tenant, + 'user': User, + 'ec2': Ec2, + } + + +def print_commands(cmds): + print + print "Available commands:" + o = [] + max_length = max([len(k) for k in cmds]) + 2 + for k, cmd in sorted(cmds.iteritems()): + initial_indent = '%s%s: ' % (' ' * (max_length - len(k)), k) + tw = textwrap.TextWrapper(initial_indent=initial_indent, + subsequent_indent=' ' * (max_length + 2), + width=80) + o.extend(tw.wrap( + (cmd.__doc__ and cmd.__doc__ or 'no docs').strip().split('\n')[0])) + print '\n'.join(o) + + +def run(cmd, args): + return CMDS[cmd](argv=args).run() + + +def main(argv=None, config_files=None): + args = CONF(config_files=config_files, args=argv) + if len(args) < 2: + CONF.print_help() + print_commands(CMDS) + sys.exit(1) + + cmd = args[1] + if cmd in CMDS: + return run(cmd, (args[:1] + args[2:])) From 608b9a270bf2bdc017e02d2575669ec62d0645e7 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 20:29:33 -0800 Subject: [PATCH 248/334] remove this useless catalog --- ...keystone_compat_diablo_sample_catalog.json | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 tests/keystone_compat_diablo_sample_catalog.json diff --git a/tests/keystone_compat_diablo_sample_catalog.json b/tests/keystone_compat_diablo_sample_catalog.json deleted file mode 100644 index 1ccef4a1ed..0000000000 --- a/tests/keystone_compat_diablo_sample_catalog.json +++ /dev/null @@ -1,53 +0,0 @@ -[{ - "name":"Cloud Servers", - "type":"compute", - "endpoints":[{ - "tenantId":"1", - "publicURL":"https://compute.north.host/v1/1234", - "internalURL":"https://compute.north.host/v1/1234", - "region":"North", - "versionId":"1.0", - "versionInfo":"https://compute.north.host/v1.0/", - "versionList":"https://compute.north.host/" - }, - { - "tenantId":"2", - "publicURL":"https://compute.north.host/v1.1/3456", - "internalURL":"https://compute.north.host/v1.1/3456", - "region":"North", - "versionId":"1.1", - "versionInfo":"https://compute.north.host/v1.1/", - "versionList":"https://compute.north.host/" - } - ], - "endpoints_links":[] -}, -{ - "name":"Cloud Files", - "type":"object-store", - "endpoints":[{ - "tenantId":"11", - "publicURL":"https://compute.north.host/v1/blah-blah", - "internalURL":"https://compute.north.host/v1/blah-blah", - "region":"South", - "versionId":"1.0", - "versionInfo":"uri", - "versionList":"uri" - }, - { - "tenantId":"2", - "publicURL":"https://compute.north.host/v1.1/blah-blah", - "internalURL":"https://compute.north.host/v1.1/blah-blah", - "region":"South", - "versionId":"1.1", - "versionInfo":"https://compute.north.host/v1.1/", - "versionList":"https://compute.north.host/" - } - ], - "endpoints_links":[{ - "rel":"next", - "href":"https://identity.north.host/v2.0/endpoints?marker=2" - } - ] -} -] From d4f2bf5fdefca433ee81075dd28be6f1b7387b50 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 Jan 2012 23:01:51 -0800 Subject: [PATCH 249/334] add a bunch of basic tests for the cli --- keystone/cli.py | 146 ++++++++++++++++++++++++++++++++++- keystone/common/wsgi.py | 1 - keystone/test.py | 1 - tests/test_cli.py | 61 +++++++++++++++ tests/test_keystoneclient.py | 80 ++++++++++--------- 5 files changed, 249 insertions(+), 40 deletions(-) create mode 100644 tests/test_cli.py diff --git a/keystone/cli.py b/keystone/cli.py index 814ef51304..0469ab3a65 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -1,8 +1,10 @@ from __future__ import absolute_import +import json import logging import os import sys +import StringIO import textwrap import cli.app @@ -19,7 +21,7 @@ config.register_cli_str('endpoint', default='http://localhost:$admin_port/v2.0', #group='ks', conf=CONF) -config.register_cli_str('auth_token', +config.register_cli_str('auth-token', default='$admin_token', #group='ks', help='asdasd', @@ -113,7 +115,29 @@ class ClientCommand(BaseApp): if CONF.id_only and getattr(resp, 'id'): print resp.id return - print resp + + if resp is None: + return + + # NOTE(termie): this is ugly but it is mostly because the keystoneclient + # code doesn't give us very serializable instance objects + if type(resp) in [type(list()), type(tuple())]: + o = [] + for r in resp: + d = {} + for k, v in sorted(r.__dict__.iteritems()): + if k[0] == '_' or k == 'manager': + continue + d[k] = v + o.append(d) + else: + o = {} + for k, v in sorted(resp.__dict__.iteritems()): + if k[0] == '_' or k == 'manager': + continue + o[k] = v + + print json.dumps(o) def print_help(self): CONF.set_usage(CONF.usage.replace( @@ -175,6 +199,122 @@ CMDS = {'db_sync': DbSync, } +class CommandLineGenerator(object): + """A keystoneclient lookalike to generate keystone-manage commands. + + One would use it like so: + + >>> gen = CommandLineGenerator(id_only=None) + >>> cl = gen.ec2.create(user_id='foo', tenant_id='foo') + >>> cl.to_argv() + ... ['keystone-manage', + '--id-only', + 'ec2', + 'create', + 'user_id=foo', + 'tenant_id=foo'] + + """ + + cmd = 'keystone-manage' + + def __init__(self, cmd=None, execute=False, **kw): + if cmd: + self.cmd = cmd + self.flags = kw + self.execute = execute + + def __getattr__(self, key): + return _Manager(self, key) + + +class _Manager(object): + def __init__(self, parent, name): + self.parent = parent + self.name = name + + def __getattr__(self, key): + return _CommandLine(cmd=self.parent.cmd, + flags=self.parent.flags, + manager=self.name, + method=key, + execute=self.parent.execute) + + +class _CommandLine(object): + def __init__(self, cmd, flags, manager, method, execute=False): + self.cmd = cmd + self.flags = flags + self.manager = manager + self.method = method + self.execute = execute + self.kw = {} + + def __call__(self, **kw): + self.kw = kw + if self.execute: + logging.debug('generated cli: %s', str(self)) + out = StringIO.StringIO() + old_out = sys.stdout + sys.stdout = out + try: + main(self.to_argv()) + except SystemExit as e: + pass + finally: + sys.stdout = old_out + rv = out.getvalue().strip().split('\n')[-1] + try: + loaded = json.loads(rv) + if type(loaded) in [type(list()), type(tuple())]: + return [DictWrapper(**x) for x in loaded] + elif type(loaded) is type(dict()): + return DictWrapper(**loaded) + except Exception: + logging.exception('Could not parse JSON: %s', rv) + return rv + return self + + def __flags(self): + o = [] + for k, v in self.flags.iteritems(): + k = k.replace('_', '-') + if v is None: + o.append('--%s' % k) + else: + o.append('--%s=%s' % (k, str(v))) + return o + + def __manager(self): + if self.manager.endswith('s'): + return self.manager[:-1] + return self.manager + + def __kw(self): + o = [] + for k, v in self.kw.iteritems(): + o.append('%s=%s' % (k, str(v))) + return o + + def to_argv(self): + return ([self.cmd] + + self.__flags() + + [self.__manager(), self.method] + + self.__kw()) + + def __str__(self): + args = self.to_argv() + return ' '.join(args[:1] + ['"%s"' % x for x in args[1:]]) + + +class DictWrapper(dict): + def __getattr__(self, key): + try: + return self[key] + except KeyError: + raise AttributeError(key) + + def print_commands(cmds): print print "Available commands:" @@ -195,7 +335,9 @@ def run(cmd, args): def main(argv=None, config_files=None): + CONF.reset() args = CONF(config_files=config_files, args=argv) + if len(args) < 2: CONF.print_help() print_commands(CMDS) diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index fafa969435..34caec2c4d 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -196,7 +196,6 @@ class Application(BaseApplication): creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') - print creds # Accept either is_admin or the admin role assert self.policy_api.can_haz(context, ('is_admin:1', 'roles:admin'), diff --git a/keystone/test.py b/keystone/test.py index 9e1137a7a5..4265514244 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -146,7 +146,6 @@ class TestCase(unittest.TestCase): for tenant_id in tenants: self.identity_api.add_user_to_tenant(tenant_id, user['id']) setattr(self, 'user_%s' % user['id'], user_copy) - print user_copy for role in fixtures.ROLES: rv = self.identity_api.create_role(role['id'], role) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000000..82fde86df8 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,61 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +import nose.exc + +from keystone import config +from keystone import test +from keystone.common import utils + +import default_fixtures +import test_keystoneclient + +CONF = config.CONF +KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' + + +class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): + def setUp(self): + super(CliMasterTestCase, self).setUp() + from keystone import cli + self.cli = cli + + def get_client(self, user_ref=None, tenant_ref=None): + if user_ref is None: + user_ref = self.user_foo + if tenant_ref is None: + for user in default_fixtures.USERS: + if user['id'] == user_ref['id']: + tenant_id = user['tenants'][0] + else: + tenant_id = tenant_ref['id'] + + cl = self._client(username=user_ref['name'], + password=user_ref['password'], + tenant_id=tenant_id) + gen = self.cli.CommandLineGenerator( + cmd=test.rootdir('bin', 'keystone-manage'), + execute=True, + auth_token=cl.auth_token, + endpoint=cl.management_url) + gen.auth_token = cl.auth_token + gen.management_url = cl.management_url + return gen + + def test_authenticate_tenant_id_and_tenants(self): + raise nose.exc.SkipTest('N/A') + + def test_authenticate_token_no_tenant(self): + raise nose.exc.SkipTest('N/A') + + def test_authenticate_token_tenant_id(self): + raise nose.exc.SkipTest('N/A') + + def test_authenticate_token_tenant_name(self): + raise nose.exc.SkipTest('N/A') + + def test_tenant_create_update_and_delete(self): + raise nose.exc.SkipTest('cli does not support booleans yet') + def test_invalid_password(self): + raise nose.exc.SkipTest('N/A') + + def test_user_create_update_delete(self): + raise nose.exc.SkipTest('cli does not support booleans yet') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index d38313d303..6265c17510 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -113,7 +113,7 @@ class KcMasterTestCase(CompatTestCase): def test_endpoints(self): client = self.get_client() token = client.auth_token - endpoints = client.tokens.endpoints(token) + endpoints = client.tokens.endpoints(token=token) # FIXME(ja): this test should require the "keystone:admin" roled # (probably the role set via --keystone_admin_role flag) @@ -125,16 +125,16 @@ class KcMasterTestCase(CompatTestCase): test_tenant = 'new_tenant' client = self.get_client() - tenant = client.tenants.create(test_tenant, + tenant = client.tenants.create(tenant_name=test_tenant, description="My new tenant!", enabled=True) self.assertEquals(tenant.name, test_tenant) - tenant = client.tenants.get(tenant.id) + tenant = client.tenants.get(tenant_id=tenant.id) self.assertEquals(tenant.name, test_tenant) # TODO(devcamcar): update gives 404. why? - tenant = client.tenants.update(tenant.id, + tenant = client.tenants.update(tenant_id=tenant.id, tenant_name='new_tenant2', enabled=False, description='new description') @@ -142,7 +142,7 @@ class KcMasterTestCase(CompatTestCase): self.assertFalse(tenant.enabled) self.assertEquals(tenant.description, 'new description') - client.tenants.delete(tenant.id) + client.tenants.delete(tenant=tenant.id) self.assertRaises(client_exceptions.NotFound, client.tenants.get, tenant.id) @@ -153,25 +153,26 @@ class KcMasterTestCase(CompatTestCase): def test_tenant_add_and_remove_user(self): client = self.get_client() - client.roles.add_user_to_tenant(self.tenant_baz['id'], - self.user_foo['id'], - self.role_useless['id']) + client.roles.add_user_to_tenant(tenant_id=self.tenant_baz['id'], + user_id=self.user_foo['id'], + role_id=self.role_useless['id']) tenant_refs = client.tenants.list() self.assert_(self.tenant_baz['id'] in [x.id for x in tenant_refs]) # get the "role_refs" so we get the proper id, this is how the clients # do it - roleref_refs = client.roles.get_user_role_refs(self.user_foo['id']) + roleref_refs = client.roles.get_user_role_refs( + user_id=self.user_foo['id']) for roleref_ref in roleref_refs: if (roleref_ref.roleId == self.role_useless['id'] and roleref_ref.tenantId == self.tenant_baz['id']): # use python's scope fall through to leave roleref_ref set break - client.roles.remove_user_from_tenant(self.tenant_baz['id'], - self.user_foo['id'], - roleref_ref.id) + client.roles.remove_user_from_tenant(tenant_id=self.tenant_baz['id'], + user_id=self.user_foo['id'], + role_id=roleref_ref.id) tenant_refs = client.tenants.list() self.assert_(self.tenant_baz['id'] not in @@ -194,17 +195,19 @@ class KcMasterTestCase(CompatTestCase): test_username = 'new_user' client = self.get_client() - user = client.users.create(test_username, 'password', 'user1@test.com') + user = client.users.create(name=test_username, + password='password', + email='user1@test.com') self.assertEquals(user.name, test_username) - user = client.users.get(user.id) + user = client.users.get(user=user.id) self.assertEquals(user.name, test_username) - user = client.users.update_email(user, 'user2@test.com') + user = client.users.update_email(user=user, email='user2@test.com') self.assertEquals(user.email, 'user2@test.com') # NOTE(termie): update_enabled doesn't return anything, probably a bug - client.users.update_enabled(user, False) + client.users.update_enabled(user=user, enabled=False) user = client.users.get(user.id) self.assertFalse(user.enabled) @@ -214,12 +217,12 @@ class KcMasterTestCase(CompatTestCase): password='password') client.users.update_enabled(user, True) - user = client.users.update_password(user, 'password2') + user = client.users.update_password(user=user, password='password2') test_client = self._client(username=test_username, password='password2') - user = client.users.update_tenant(user, 'bar') + user = client.users.update_tenant(user=user, tenant='bar') # TODO(ja): once keystonelight supports default tenant # when you login without specifying tenant, the # token should be scoped to tenant 'bar' @@ -237,12 +240,12 @@ class KcMasterTestCase(CompatTestCase): def test_user_get(self): client = self.get_client() - user = client.users.get(self.user_foo['id']) + user = client.users.get(user=self.user_foo['id']) self.assertRaises(AttributeError, lambda: user.password) def test_role_get(self): client = self.get_client() - role = client.roles.get('keystone_admin') + role = client.roles.get(role='keystone_admin') self.assertEquals(role.id, 'keystone_admin') def test_role_create_and_delete(self): @@ -250,16 +253,16 @@ class KcMasterTestCase(CompatTestCase): test_role = 'new_role' client = self.get_client() - role = client.roles.create(test_role) + role = client.roles.create(name=test_role) self.assertEquals(role.name, test_role) - role = client.roles.get(role) + role = client.roles.get(role=role.id) self.assertEquals(role.name, test_role) - client.roles.delete(role) + client.roles.delete(role=role.id) self.assertRaises(client_exceptions.NotFound, client.roles.get, - test_role) + role=test_role) def test_role_list(self): client = self.get_client() @@ -269,25 +272,26 @@ class KcMasterTestCase(CompatTestCase): def test_roles_get_by_user(self): client = self.get_client() - roles = client.roles.get_user_role_refs('foo') + roles = client.roles.get_user_role_refs(user_id='foo') self.assertTrue(len(roles) > 0) def test_ec2_credential_crud(self): client = self.get_client() - creds = client.ec2.list(self.user_foo['id']) + creds = client.ec2.list(user_id=self.user_foo['id']) self.assertEquals(creds, []) - cred = client.ec2.create(self.user_foo['id'], self.tenant_bar['id']) - creds = client.ec2.list(self.user_foo['id']) + cred = client.ec2.create(user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + creds = client.ec2.list(user_id=self.user_foo['id']) self.assertEquals(creds, [cred]) - got = client.ec2.get(self.user_foo['id'], cred.access) + got = client.ec2.get(user_id=self.user_foo['id'], access=cred.access) self.assertEquals(cred, got) # FIXME(ja): need to test ec2 validation here - client.ec2.delete(self.user_foo['id'], cred.access) - creds = client.ec2.list(self.user_foo['id']) + client.ec2.delete(user_id=self.user_foo['id'], access=cred.access) + creds = client.ec2.list(user_id=self.user_foo['id']) self.assertEquals(creds, []) def test_ec2_credentials_list_unauthorized_user(self): @@ -329,20 +333,24 @@ class KcMasterTestCase(CompatTestCase): test_service = 'new_service' client = self.get_client() - service = client.services.create(test_service, 'test', 'test') + service = client.services.create(name=test_service, + service_type='test', + description='test') self.assertEquals(service.name, test_service) - service = client.services.get(service.id) + service = client.services.get(id=service.id) self.assertEquals(service.name, test_service) - client.services.delete(service.id) + client.services.delete(id=service.id) self.assertRaises(client_exceptions.NotFound, client.services.get, - service.id) + id=service.id) def test_service_list(self): client = self.get_client() test_service = 'new_service' - service = client.services.create(test_service, 'test', 'test') + service = client.services.create(name=test_service, + service_type='test', + description='test') services = client.services.list() # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(services) > 0) From fcea15df2a1dba5ae65f90a16e5f865a08d00457 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 25 Jan 2012 16:50:14 -0800 Subject: [PATCH 250/334] Add s3tokens validation. There is still some works to be done in the swift middleware but this should works. Documentation and tests are in the planning. --- keystone/contrib/s3/__init__.py | 1 + keystone/contrib/s3/core.py | 98 +++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 keystone/contrib/s3/__init__.py create mode 100644 keystone/contrib/s3/core.py diff --git a/keystone/contrib/s3/__init__.py b/keystone/contrib/s3/__init__.py new file mode 100644 index 0000000000..2214959601 --- /dev/null +++ b/keystone/contrib/s3/__init__.py @@ -0,0 +1 @@ +from keystone.contrib.s3.core import * diff --git a/keystone/contrib/s3/core.py b/keystone/contrib/s3/core.py new file mode 100644 index 0000000000..780fe9e761 --- /dev/null +++ b/keystone/contrib/s3/core.py @@ -0,0 +1,98 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +"""Main entry point into the S3 Credentials service. + +TODO-DOCS +""" + +import uuid +import base64 +import hmac + +from hashlib import sha1 +from keystone import catalog +from keystone import config +from keystone import identity +from keystone import policy +from keystone import token +from keystone import service +from keystone.common import wsgi +from keystone.contrib.ec2 import Manager as EC2Manager + +CONF = config.CONF + + +class S3Extension(wsgi.ExtensionRouter): + def add_routes(self, mapper): + s3_controller = S3Controller() + # validation + mapper.connect('/s3tokens', + controller=s3_controller, + action='authenticate_s3', + conditions=dict(method=['POST'])) + + # No need CRUD stuff since we are sharing keystone.contrib.ec2 + # infos. + + +class S3Controller(wsgi.Application): + def __init__(self): + self.catalog_api = catalog.Manager() + self.identity_api = identity.Manager() + self.token_api = token.Manager() + self.policy_api = policy.Manager() + self.ec2_api = EC2Manager() + super(S3Controller, self).__init__() + + def authenticate_s3(self, context, credentials=None): + """Validate a signed S3 request and provide a token. + + TODO-DOCS:: + + :param context: standard context + :param credentials: dict of s3 signature + :returns: token: openstack token equivalent to access key along + with the corresponding service catalog and roles + """ + + creds_ref = self.ec2_api.get_credential(context, + credentials['access']) + + msg = base64.urlsafe_b64decode(str(credentials['token'])) + key = str(creds_ref['secret']) + s = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() + signature = credentials['signature'] + if signature == s: + pass + else: + raise Exception("Not Authorized") + + token_id = uuid.uuid4().hex + tenant_ref = self.identity_api.get_tenant(context, + creds_ref['tenant_id']) + user_ref = self.identity_api.get_user(context, + creds_ref['user_id']) + metadata_ref = self.identity_api.get_metadata( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id']) + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=user_ref['id'], + tenant_id=tenant_ref['id'], + metadata=metadata_ref) + + token_ref = self.token_api.create_token( + context, token_id, dict(expires='', + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref)) + + roles_ref = [] + for role_id in metadata_ref.get('roles', []): + roles_ref.append(self.identity_api.get_role(context, role_id)) + + token_controller = service.TokenController() + return token_controller._format_authenticate( + token_ref, roles_ref, catalog_ref) From 1efee11f587a73969c68a6c77fc97d786a1903e6 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 25 Jan 2012 19:33:43 -0600 Subject: [PATCH 251/334] add tests that auth with tenant user isn't member of --- tests/default_fixtures.py | 1 + tests/test_keystoneclient.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/default_fixtures.py b/tests/default_fixtures.py index 22ac95f0bf..6d455bdf66 100644 --- a/tests/default_fixtures.py +++ b/tests/default_fixtures.py @@ -3,6 +3,7 @@ TENANTS = [ {'id': 'baz', 'name': 'BAZ'}, ] +# NOTE(ja): a role of keystone_admin and attribute "is_admin" is done in setUp USERS = [ {'id': 'foo', 'name': 'FOO', 'password': 'foo2', 'tenants': ['bar',]}, {'id': 'two', 'name': 'TWO', 'password': 'two2', 'tenants': ['baz',]}, diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 6265c17510..477aa3046e 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -88,6 +88,14 @@ class KcMasterTestCase(CompatTestCase): tenants = client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + def test_authenticate_invalid_tenant_id(self): + from keystoneclient import exceptions as client_exceptions + self.assertRaises(client_exceptions.AuthorizationFailure, + self._client, + username=self.user_foo['name'], + password=self.user_foo['password'], + tenant_id='baz') + def test_authenticate_token_no_tenant(self): client = self.get_client() token = client.auth_token @@ -102,6 +110,13 @@ class KcMasterTestCase(CompatTestCase): tenants = token_client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + def test_authenticate_token_invalid_tenant_id(self): + from keystoneclient import exceptions as client_exceptions + client = self.get_client() + token = client.auth_token + self.assertRaises(client_exceptions.AuthorizationFailure, + self._client, token=token, tenant_id='baz') + def test_authenticate_token_tenant_name(self): client = self.get_client() token = client.auth_token @@ -219,8 +234,8 @@ class KcMasterTestCase(CompatTestCase): user = client.users.update_password(user=user, password='password2') - test_client = self._client(username=test_username, - password='password2') + self._client(username=test_username, + password='password2') user = client.users.update_tenant(user=user, tenant='bar') # TODO(ja): once keystonelight supports default tenant From b1cd21426cc12ab65a89f617b9a8d09322dde009 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 26 Jan 2012 01:37:37 +0000 Subject: [PATCH 252/334] Simplify code. Per Termie's suggestion, share as much as possible with ec2 module and instantiate s3 module from it. --- keystone/contrib/ec2/core.py | 52 ++++++++++++--------- keystone/contrib/s3/core.py | 91 +++++++----------------------------- 2 files changed, 49 insertions(+), 94 deletions(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index fab5f3891b..d18a73a8ca 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -26,8 +26,10 @@ from keystone import catalog from keystone import config from keystone import identity from keystone import policy +from keystone import service from keystone import token from keystone.common import manager +from keystone.common import utils from keystone.common import wsgi @@ -52,7 +54,7 @@ class Ec2Extension(wsgi.ExtensionRouter): # validation mapper.connect('/ec2tokens', controller=ec2_controller, - action='authenticate_ec2', + action='authenticate', conditions=dict(method=['POST'])) # crud @@ -83,7 +85,24 @@ class Ec2Controller(wsgi.Application): self.ec2_api = Manager() super(Ec2Controller, self).__init__() - def authenticate_ec2(self, context, credentials=None, + def check_signature(self, creds_ref, credentials): + signer = utils.Signer(creds_ref['secret']) + signature = signer.generate(credentials) + if signature == credentials['signature']: + return + # NOTE(vish): Some libraries don't use the port when signing + # requests, so try again without port. + elif ':' in credentials['signature']: + hostname, _port = credentials['host'].split(":") + credentials['host'] = hostname + signature = signer.generate(credentials) + if signature != credentials.signature: + # TODO(termie): proper exception + raise Exception("Not Authorized") + else: + raise Exception("Not Authorized") + + def authenticate(self, context, credentials=None, ec2Credentials=None): """Validate a signed EC2 request and provide a token. @@ -113,27 +132,17 @@ class Ec2Controller(wsgi.Application): creds_ref = self.ec2_api.get_credential(context, credentials['access']) - signer = utils.Signer(creds_ref['secret']) - signature = signer.generate(credentials) - if signature == credentials['signature']: - pass - # NOTE(vish): Some libraries don't use the port when signing - # requests, so try again without port. - elif ':' in credentials['signature']: - hostname, _port = credentials['host'].split(":") - credentials['host'] = hostname - signature = signer.generate(credentials) - if signature != credentials.signature: - # TODO(termie): proper exception - raise Exception("Not Authorized") - else: - raise Exception("Not Authorized") + self.check_signature(creds_ref, credentials) # TODO(termie): don't create new tokens every time # TODO(termie): this is copied from TokenController.authenticate token_id = uuid.uuid4().hex - tenant_ref = self.identity_api.get_tenant(creds_ref['tenant_id']) - user_ref = self.identity_api.get_user(creds_ref['user_id']) + tenant_ref = self.identity_api.get_tenant( + context=context, + tenant_id=creds_ref['tenant_id']) + user_ref = self.identity_api.get_user( + context=context, + user_id=creds_ref['user_id']) metadata_ref = self.identity_api.get_metadata( context=context, user_id=user_ref['id'], @@ -162,8 +171,9 @@ class Ec2Controller(wsgi.Application): # TODO(termie): i don't think the ec2 middleware currently expects a # full return, but it contains a note saying that it # would be better to expect a full return - return TokenController._format_authenticate( - self, token_ref, roles_ref, catalog_ref) + token_controller = service.TokenController() + return token_controller._format_authenticate( + token_ref, roles_ref, catalog_ref) def create_credential(self, context, user_id, tenant_id): """Create a secret/access pair for use with ec2 style auth. diff --git a/keystone/contrib/s3/core.py b/keystone/contrib/s3/core.py index 780fe9e761..9b310e4d83 100644 --- a/keystone/contrib/s3/core.py +++ b/keystone/contrib/s3/core.py @@ -5,94 +5,39 @@ TODO-DOCS """ -import uuid import base64 import hmac from hashlib import sha1 -from keystone import catalog + from keystone import config -from keystone import identity -from keystone import policy -from keystone import token -from keystone import service from keystone.common import wsgi -from keystone.contrib.ec2 import Manager as EC2Manager +from keystone.contrib import ec2 CONF = config.CONF +def check_signature(creds_ref, credentials): + signature = credentials['signature'] + msg = base64.urlsafe_b64decode(str(credentials['token'])) + key = str(creds_ref['secret']) + signed = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() + + if signature == signed: + pass + else: + raise Exception("Not Authorized") + + class S3Extension(wsgi.ExtensionRouter): def add_routes(self, mapper): - s3_controller = S3Controller() + controller = ec2.Ec2Controller() + controller.check_signature = check_signature # validation mapper.connect('/s3tokens', - controller=s3_controller, - action='authenticate_s3', + controller=controller, + action='authenticate', conditions=dict(method=['POST'])) # No need CRUD stuff since we are sharing keystone.contrib.ec2 # infos. - - -class S3Controller(wsgi.Application): - def __init__(self): - self.catalog_api = catalog.Manager() - self.identity_api = identity.Manager() - self.token_api = token.Manager() - self.policy_api = policy.Manager() - self.ec2_api = EC2Manager() - super(S3Controller, self).__init__() - - def authenticate_s3(self, context, credentials=None): - """Validate a signed S3 request and provide a token. - - TODO-DOCS:: - - :param context: standard context - :param credentials: dict of s3 signature - :returns: token: openstack token equivalent to access key along - with the corresponding service catalog and roles - """ - - creds_ref = self.ec2_api.get_credential(context, - credentials['access']) - - msg = base64.urlsafe_b64decode(str(credentials['token'])) - key = str(creds_ref['secret']) - s = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() - signature = credentials['signature'] - if signature == s: - pass - else: - raise Exception("Not Authorized") - - token_id = uuid.uuid4().hex - tenant_ref = self.identity_api.get_tenant(context, - creds_ref['tenant_id']) - user_ref = self.identity_api.get_user(context, - creds_ref['user_id']) - metadata_ref = self.identity_api.get_metadata( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id']) - catalog_ref = self.catalog_api.get_catalog( - context=context, - user_id=user_ref['id'], - tenant_id=tenant_ref['id'], - metadata=metadata_ref) - - token_ref = self.token_api.create_token( - context, token_id, dict(expires='', - id=token_id, - user=user_ref, - tenant=tenant_ref, - metadata=metadata_ref)) - - roles_ref = [] - for role_id in metadata_ref.get('roles', []): - roles_ref.append(self.identity_api.get_role(context, role_id)) - - token_controller = service.TokenController() - return token_controller._format_authenticate( - token_ref, roles_ref, catalog_ref) From d5443e2ef0ac8d1c33ba3644ddb9053b68a6ed0d Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 26 Jan 2012 00:26:30 -0600 Subject: [PATCH 253/334] initial stab at requiring adminness --- tests/test_keystoneclient.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 477aa3046e..cb53170415 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -369,3 +369,41 @@ class KcMasterTestCase(CompatTestCase): services = client.services.list() # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(services) > 0) + + def test_admin_requires_adminness(self): + from keystoneclient import exceptions as client_exceptions + # FIXME(termie): this should be Unauthorized + exception = client_exceptions.ClientException + + two = self.get_client(self.user_two) # non-admin user + + # USER CRUD + self.assertRaises(exception, two.users.list) + self.assertRaises(exception, two.users.get, self.user_two['id']) + self.assertRaises(exception, two.users.create, name='oops', + password='password', email='oops@test.com') + self.assertRaises(exception, two.users.delete, self.user_foo['id']) + + # TENANT CRUD + # NOTE(ja): tenants.list is different since /tenants fulfills the + # two different tasks: return list of all tenants & return + # list of tenants the current user is a member of... + # which means if you are admin getting the list + # of tenants for admin user is annoying? + tenants = two.tenants.list() + self.assertTrue(len(tenants) == 1) + self.assertTrue(tenants[0].id == self.tenant_baz['id']) + self.assertRaises(exception, two.tenants.get, self.tenant_bar['id']) + self.assertRaises(exception, two.tenants.create, + tenant_name='oops', description="shouldn't work!", + enabled=True) + self.assertRaises(exception, two.tenants.delete, self.tenant_baz['id']) + + # ROLE CRUD + self.assertRaises(exception, two.roles.get, role='keystone_admin') + self.assertRaises(exception, two.roles.list) + self.assertRaises(exception, two.roles.create, name='oops') + self.assertRaises(exception, two.roles.delete, name='keystone_admin') + + # TODO(ja): MEMBERSHIP CRUD + # TODO(ja): determine what else todo From 726b5adab004d84cec061b1de2ab6fe306b08973 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 26 Jan 2012 00:30:24 -0600 Subject: [PATCH 254/334] invalid params for roles.delete --- tests/test_keystoneclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index cb53170415..18b05af3fe 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -403,7 +403,7 @@ class KcMasterTestCase(CompatTestCase): self.assertRaises(exception, two.roles.get, role='keystone_admin') self.assertRaises(exception, two.roles.list) self.assertRaises(exception, two.roles.create, name='oops') - self.assertRaises(exception, two.roles.delete, name='keystone_admin') + self.assertRaises(exception, two.roles.delete, 'keystone_admin') # TODO(ja): MEMBERSHIP CRUD # TODO(ja): determine what else todo From d6d56e45dc416cbdb000266e2a3ad3a3880fd166 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 26 Jan 2012 15:04:54 -0600 Subject: [PATCH 255/334] fix style and termie's comments about comments --- tests/test_keystoneclient.py | 49 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 18b05af3fe..9766108ad4 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -372,17 +372,25 @@ class KcMasterTestCase(CompatTestCase): def test_admin_requires_adminness(self): from keystoneclient import exceptions as client_exceptions - # FIXME(termie): this should be Unauthorized + # FIXME(ja): this should be Unauthorized exception = client_exceptions.ClientException two = self.get_client(self.user_two) # non-admin user # USER CRUD - self.assertRaises(exception, two.users.list) - self.assertRaises(exception, two.users.get, self.user_two['id']) - self.assertRaises(exception, two.users.create, name='oops', - password='password', email='oops@test.com') - self.assertRaises(exception, two.users.delete, self.user_foo['id']) + self.assertRaises(exception, + two.users.list) + self.assertRaises(exception, + two.users.get, + user=self.user_two['id']) + self.assertRaises(exception, + two.users.create, + name='oops', + password='password', + email='oops@test.com') + self.assertRaises(exception, + two.users.delete, + user=self.user_foo['id']) # TENANT CRUD # NOTE(ja): tenants.list is different since /tenants fulfills the @@ -393,17 +401,30 @@ class KcMasterTestCase(CompatTestCase): tenants = two.tenants.list() self.assertTrue(len(tenants) == 1) self.assertTrue(tenants[0].id == self.tenant_baz['id']) - self.assertRaises(exception, two.tenants.get, self.tenant_bar['id']) - self.assertRaises(exception, two.tenants.create, - tenant_name='oops', description="shouldn't work!", + self.assertRaises(exception, + two.tenants.get, + tenant_id=self.tenant_bar['id']) + self.assertRaises(exception, + two.tenants.create, + tenant_name='oops', + description="shouldn't work!", enabled=True) - self.assertRaises(exception, two.tenants.delete, self.tenant_baz['id']) + self.assertRaises(exception, + two.tenants.delete, + tenant=self.tenant_baz['id']) # ROLE CRUD - self.assertRaises(exception, two.roles.get, role='keystone_admin') - self.assertRaises(exception, two.roles.list) - self.assertRaises(exception, two.roles.create, name='oops') - self.assertRaises(exception, two.roles.delete, 'keystone_admin') + self.assertRaises(exception, + two.roles.get, + role='keystone_admin') + self.assertRaises(exception, + two.roles.list) + self.assertRaises(exception, + two.roles.create, + name='oops') + self.assertRaises(exception, + two.roles.delete, + role='keystone_admin') # TODO(ja): MEMBERSHIP CRUD # TODO(ja): determine what else todo From 3974760a4406060061017f03bb7eabe5b1937a23 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 26 Jan 2012 13:29:38 -0800 Subject: [PATCH 256/334] Make it as a subclass. as advised by termie make it as a subclass instead of patching the method. --- keystone/contrib/s3/core.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/keystone/contrib/s3/core.py b/keystone/contrib/s3/core.py index 9b310e4d83..208c6aa555 100644 --- a/keystone/contrib/s3/core.py +++ b/keystone/contrib/s3/core.py @@ -17,27 +17,21 @@ from keystone.contrib import ec2 CONF = config.CONF -def check_signature(creds_ref, credentials): - signature = credentials['signature'] - msg = base64.urlsafe_b64decode(str(credentials['token'])) - key = str(creds_ref['secret']) - signed = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() - - if signature == signed: - pass - else: - raise Exception("Not Authorized") - - class S3Extension(wsgi.ExtensionRouter): def add_routes(self, mapper): - controller = ec2.Ec2Controller() - controller.check_signature = check_signature + controller = S3Controller() # validation mapper.connect('/s3tokens', controller=controller, action='authenticate', conditions=dict(method=['POST'])) - # No need CRUD stuff since we are sharing keystone.contrib.ec2 - # infos. + +class S3Controller(ec2.Ec2Controller): + def check_signature(self, creds_ref, credentials): + msg = base64.urlsafe_b64decode(str(credentials['token'])) + key = str(creds_ref['secret']) + signed = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() + + if credentials['signature'] != signed: + raise Exception("Not Authorized") From 103fc87b8eb80eae4f20e23e9ad6485a33f360f7 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 26 Jan 2012 14:29:47 -0800 Subject: [PATCH 257/334] indents. --- keystone/contrib/ec2/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index d18a73a8ca..08e46647bc 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -138,11 +138,11 @@ class Ec2Controller(wsgi.Application): # TODO(termie): this is copied from TokenController.authenticate token_id = uuid.uuid4().hex tenant_ref = self.identity_api.get_tenant( - context=context, - tenant_id=creds_ref['tenant_id']) + context=context, + tenant_id=creds_ref['tenant_id']) user_ref = self.identity_api.get_user( - context=context, - user_id=creds_ref['user_id']) + context=context, + user_id=creds_ref['user_id']) metadata_ref = self.identity_api.get_metadata( context=context, user_id=user_ref['id'], @@ -173,7 +173,7 @@ class Ec2Controller(wsgi.Application): # would be better to expect a full return token_controller = service.TokenController() return token_controller._format_authenticate( - token_ref, roles_ref, catalog_ref) + token_ref, roles_ref, catalog_ref) def create_credential(self, context, user_id, tenant_id): """Create a secret/access pair for use with ec2 style auth. From 080f523ff771c861251f1c33e32ec9ff2d358f88 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Fri, 27 Jan 2012 07:00:44 +0000 Subject: [PATCH 258/334] fixing up PIP requirements for testing and virtualenv --- tools/pip-requires | 2 +- tools/pip-requires-test | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/pip-requires b/tools/pip-requires index d4157f8ef1..c075d369ee 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -4,7 +4,7 @@ eventlet==0.9.12 PasteDeploy paste routes -pycli +pyCLI sqlalchemy sqlalchemy-migrate py-bcrypt diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 563eaa2cbc..e40795c826 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -5,8 +5,10 @@ eventlet==0.9.12 PasteDeploy paste routes +pyCLI sqlalchemy sqlalchemy-migrate +py-bcrypt # keystonelight testing dependencies nose @@ -15,6 +17,7 @@ nosexcover # for python-keystoneclient httplib2 pep8 +-e git+https://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient # for python-novaclient prettytable From 68aa9cd10fca548dcbf2a8a58d7cf902a1af655b Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Fri, 27 Jan 2012 07:06:03 +0000 Subject: [PATCH 259/334] adding python keystoneclient to setup.py deps --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dcd4f2e025..6c1ce8486d 100755 --- a/setup.py +++ b/setup.py @@ -10,5 +10,5 @@ setup(name='keystone', packages=find_packages(exclude=['test', 'bin']), scripts=['bin/keystone', 'bin/keystone-manage'], zip_safe=False, - install_requires=['setuptools'], + install_requires = ['setuptools', 'python-keystoneclient'], ) From 6b38e3ceb62515f1d28078d2f36b72548023521c Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Tue, 24 Jan 2012 09:43:06 -0800 Subject: [PATCH 260/334] moving in all the original docs from keystone --- docs/source/adminAPI_curl_examples.rst | 387 +++++++++++++ docs/source/architecture.rst | 97 ++++ docs/source/backends.rst | 188 +++++++ docs/source/configuration.rst | 100 ++++ docs/source/configuringservices.rst | 333 +++++++++++ docs/source/controllingservers.rst | 288 ++++++++++ docs/source/endpoints.rst | 430 ++++++++++++++ docs/source/extensions.rst | 183 ++++++ docs/source/images/305.svg | 158 ++++++ docs/source/images/authComp.svg | 174 ++++++ docs/source/images/both.svg | 135 +++++ docs/source/images/graphs_305.svg | 41 ++ docs/source/images/graphs_authComp.svg | 48 ++ .../source/images/graphs_authCompDelegate.svg | 53 ++ docs/source/images/graphs_both.svg | 36 ++ docs/source/images/graphs_delegate_accept.svg | 52 ++ .../images/graphs_delegate_forbiden_basic.svg | 53 ++ .../images/graphs_delegate_forbiden_proxy.svg | 52 ++ .../images/graphs_delegate_reject_basic.svg | 55 ++ .../images/graphs_delegate_reject_oauth.svg | 56 ++ .../images/graphs_delegate_unimplemented.svg | 53 ++ docs/source/images/graphs_mapper.svg | 73 +++ docs/source/images/graphs_proxyAuth.svg | 51 ++ docs/source/images/graphs_separate.svg | 30 + docs/source/images/graphs_standard_accept.svg | 51 ++ docs/source/images/graphs_standard_reject.svg | 39 ++ docs/source/images/graphs_together.svg | 24 + docs/source/images/images_layouts.svg | 200 +++++++ docs/source/images/layouts.svg | 215 +++++++ docs/source/images/mapper.svg | 237 ++++++++ docs/source/images/proxyAuth.svg | 238 ++++++++ docs/source/middleware.rst | 169 ++++++ docs/source/middleware_architecture.rst | 529 ++++++++++++++++++ docs/source/migration.rst | 126 +++++ docs/source/nova-api-paste.rst | 142 +++++ docs/source/releases.rst | 36 ++ docs/source/serviceAPI_curl_examples.rst | 69 +++ docs/source/services.rst | 92 +++ docs/source/ssl.rst | 118 ++++ docs/source/usingkeystone.rst | 28 + 40 files changed, 5439 insertions(+) create mode 100644 docs/source/adminAPI_curl_examples.rst create mode 100644 docs/source/architecture.rst create mode 100644 docs/source/backends.rst create mode 100644 docs/source/configuration.rst create mode 100644 docs/source/configuringservices.rst create mode 100644 docs/source/controllingservers.rst create mode 100644 docs/source/endpoints.rst create mode 100644 docs/source/extensions.rst create mode 100644 docs/source/images/305.svg create mode 100644 docs/source/images/authComp.svg create mode 100644 docs/source/images/both.svg create mode 100644 docs/source/images/graphs_305.svg create mode 100644 docs/source/images/graphs_authComp.svg create mode 100644 docs/source/images/graphs_authCompDelegate.svg create mode 100644 docs/source/images/graphs_both.svg create mode 100644 docs/source/images/graphs_delegate_accept.svg create mode 100644 docs/source/images/graphs_delegate_forbiden_basic.svg create mode 100644 docs/source/images/graphs_delegate_forbiden_proxy.svg create mode 100644 docs/source/images/graphs_delegate_reject_basic.svg create mode 100644 docs/source/images/graphs_delegate_reject_oauth.svg create mode 100644 docs/source/images/graphs_delegate_unimplemented.svg create mode 100644 docs/source/images/graphs_mapper.svg create mode 100644 docs/source/images/graphs_proxyAuth.svg create mode 100644 docs/source/images/graphs_separate.svg create mode 100644 docs/source/images/graphs_standard_accept.svg create mode 100644 docs/source/images/graphs_standard_reject.svg create mode 100644 docs/source/images/graphs_together.svg create mode 100644 docs/source/images/images_layouts.svg create mode 100644 docs/source/images/layouts.svg create mode 100644 docs/source/images/mapper.svg create mode 100644 docs/source/images/proxyAuth.svg create mode 100644 docs/source/middleware.rst create mode 100644 docs/source/middleware_architecture.rst create mode 100644 docs/source/migration.rst create mode 100644 docs/source/nova-api-paste.rst create mode 100644 docs/source/releases.rst create mode 100644 docs/source/serviceAPI_curl_examples.rst create mode 100644 docs/source/services.rst create mode 100644 docs/source/ssl.rst create mode 100644 docs/source/usingkeystone.rst diff --git a/docs/source/adminAPI_curl_examples.rst b/docs/source/adminAPI_curl_examples.rst new file mode 100644 index 0000000000..81f96c36d3 --- /dev/null +++ b/docs/source/adminAPI_curl_examples.rst @@ -0,0 +1,387 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================= +Admin API Examples Using Curl +============================= + +These examples assume a default port value of 35357, and depend on the +``sampledata`` bundled with keystone. + +GET / +===== + +Disover API version information, links to documentation (PDF, HTML, WADL), +and supported media types:: + + $ curl http://0.0.0.0:35357 + +or:: + + $ curl http://0.0.0.0:35357/v2.0/ + +Returns:: + + { + "version":{ + "id":"v2.0", + "status":"beta", + "updated":"2011-11-19T00:00:00Z", + "links":[ + { + "rel":"self", + "href":"http://127.0.0.1:35357/v2.0/" + }, + { + "rel":"describedby", + "type":"text/html", + "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/content/" + }, + { + "rel":"describedby", + "type":"application/pdf", + "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/identity-dev-guide-2.0.pdf" + }, + { + "rel":"describedby", + "type":"application/vnd.sun.wadl+xml", + "href":"http://127.0.0.1:35357/v2.0/identity-admin.wadl" + } + ], + "media-types":[ + { + "base":"application/xml", + "type":"application/vnd.openstack.identity-v2.0+xml" + }, + { + "base":"application/json", + "type":"application/vnd.openstack.identity-v2.0+json" + } + ] + } + } + +GET /extensions +=============== + +Discover the API extensions enabled at the endpoint:: + + $ curl http://0.0.0.0:35357/extensions + +Returns:: + + { + "extensions":{ + "values":[] + } + } + +POST /tokens +============ + +Authenticate by exchanging credentials for an access token:: + + $ curl -d '{"auth":{"passwordCredentials":{"username": "joeuser", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:35357/v2.0/tokens + +Returns:: + + { + "access":{ + "token":{ + "expires":"2012-02-05T00:00:00", + "id":"887665443383838", + "tenant":{ + "id":"1", + "name":"customer-x" + } + }, + "serviceCatalog":[ + { + "endpoints":[ + { + "adminURL":"http://swift.admin-nets.local:8080/", + "region":"RegionOne", + "internalURL":"http://127.0.0.1:8080/v1/AUTH_1", + "publicURL":"http://swift.publicinternets.com/v1/AUTH_1" + } + ], + "type":"object-store", + "name":"swift" + }, + { + "endpoints":[ + { + "adminURL":"http://cdn.admin-nets.local/v1.1/1", + "region":"RegionOne", + "internalURL":"http://127.0.0.1:7777/v1.1/1", + "publicURL":"http://cdn.publicinternets.com/v1.1/1" + } + ], + "type":"object-store", + "name":"cdn" + } + ], + "user":{ + "id":"1", + "roles":[ + { + "tenantId":"1", + "id":"3", + "name":"Member" + } + ], + "name":"joeuser" + } + } + } + +.. note:: + + Take note of the value ['access']['token']['id'] value produced here (``887665443383838``, above), as you can use it in the calls below. + +GET /tokens/{token_id} +====================== + +.. note:: + + This call refers to a token known to be valid, ``887665443383838`` in this case. + +Validate a token:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 + +If the token is valid, returns:: + + { + "access":{ + "token":{ + "expires":"2012-02-05T00:00:00", + "id":"887665443383838", + "tenant":{ + "id":"1", + "name":"customer-x" + } + }, + "user":{ + "name":"joeuser", + "tenantName":"customer-x", + "id":"1", + "roles":[ + { + "serviceId":"1", + "id":"3", + "name":"Member" + } + ], + "tenantId":"1" + } + } + } + +HEAD /tokens/{token_id} +======================= + +This is a high-performance variant of the GET call documented above, which +by definition, returns no response body:: + + $ curl -I -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 + +... which returns ``200``, indicating the token is valid:: + + HTTP/1.1 200 OK + Content-Length: 0 + Content-Type: None + Date: Tue, 08 Nov 2011 23:07:44 GMT + +GET /tokens/{token_id}/endpoints +================================ + +List all endpoints for a token:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838/endpoints + +Returns:: + + { + "endpoints_links": [ + { + "href": "http://127.0.0.1:35357/tokens/887665443383838/endpoints?'marker=5&limit=10'", + "rel": "next" + } + ], + "endpoints": [ + { + "internalURL": "http://127.0.0.1:8080/v1/AUTH_1", + "name": "swift", + "adminURL": "http://swift.admin-nets.local:8080/", + "region": "RegionOne", + "tenantId": 1, + "type": "object-store", + "id": 1, + "publicURL": "http://swift.publicinternets.com/v1/AUTH_1" + }, + { + "internalURL": "http://localhost:8774/v1.0", + "name": "nova_compat", + "adminURL": "http://127.0.0.1:8774/v1.0", + "region": "RegionOne", + "tenantId": 1, + "type": "compute", + "id": 2, + "publicURL": "http://nova.publicinternets.com/v1.0/" + }, + { + "internalURL": "http://localhost:8774/v1.1", + "name": "nova", + "adminURL": "http://127.0.0.1:8774/v1.1", + "region": "RegionOne", + "tenantId": 1, + "type": "compute", + "id": 3, + "publicURL": "http://nova.publicinternets.com/v1.1/ + }, + { + "internalURL": "http://127.0.0.1:9292/v1.1/", + "name": "glance", + "adminURL": "http://nova.admin-nets.local/v1.1/", + "region": "RegionOne", + "tenantId": 1, + "type": "image", + "id": 4, + "publicURL": "http://glance.publicinternets.com/v1.1/" + }, + { + "internalURL": "http://127.0.0.1:7777/v1.1/1", + "name": "cdn", + "adminURL": "http://cdn.admin-nets.local/v1.1/1", + "region": "RegionOne", + "tenantId": 1, + "versionId": "1.1", + "versionList": "http://127.0.0.1:7777/", + "versionInfo": "http://127.0.0.1:7777/v1.1", + "type": "object-store", + "id": 5, + "publicURL": "http://cdn.publicinternets.com/v1.1/1" + } + ] + } + +GET /tenants +============ + +List all of the tenants in the system (requires an Admin ``X-Auth-Token``):: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants + +Returns:: + + { + "tenants_links": [], + "tenants": [ + { + "enabled": false, + "description": "None", + "name": "project-y", + "id": "3" + }, + { + "enabled": true, + "description": "None", + "name": "ANOTHER:TENANT", + "id": "2" + }, + { + "enabled": true, + "description": "None", + "name": "customer-x", + "id": "1" + } + ] + } + +GET /tenants/{tenant_id} +======================== + +Retrieve information about a tenant, by tenant ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1 + +Returns:: + + { + "tenant":{ + "enabled":true, + "description":"None", + "name":"customer-x", + "id":"1" + } + } + +GET /tenants/{tenant_id}/users/{user_id}/roles +============================================== + +List the roles a user has been granted on a tenant:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1/users/1/roles + +Returns:: + + { + "roles_links":[], + "roles":[ + { + "id":"3", + "name":"Member" + } + ] + } + +GET /users/{user_id} +==================== + +Retrieve information about a user, by user ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/1 + +Returns:: + + { + "user":{ + "tenantId":"1", + "enabled":true, + "id":"1", + "name":"joeuser" + } + } + +GET /users/{user_id}/roles +========================== + +Retrieve the roles granted to a user, given a user ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/4/roles + +Returns:: + + { + "roles_links":[], + "roles":[ + { + "id":"2", + "name":"KeystoneServiceAdmin" + } + ] + } diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000000..8de4550256 --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,97 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Keystone Architecture +===================== + +Keystone has two major components: Authentication and a Service Catalog. + +Authentication +-------------- + +In providing a token-based authentication service for OpenStack, keystone +has several major concepts: + +Tenant + A grouping used in OpenStack to contain relevant OpenStack services. A + tenant maps to a Nova "project-id", and in object storage, a tenant can + have multiple containers. Depending on the installation, a tenant can + represent a customer, account, organization, or project. + +User + Represents an individual within OpenStack for the purposes of + authenticating them to OpenStack services. Users have credentials, and may + be assigned to one or more tenants. When authenticated, a token is + provided that is specific to a single tenant. + +Credentials + Password or other information that uniquely identifies a User to Keystone + for the purposes of providing a token. + +Token + A token is an arbitrary bit of text that is used to share authentication + with other OpenStack services so that Keystone can provide a central + location for authenticating users for access to OpenStack services. A + token may be "scoped" or "unscoped". A scoped token represents a user + authenticated to a Tenant, where an unscoped token represents just the + user. + + Tokens are valid for a limited amount of time and may be revoked at any + time. + +Role + A role is a set of permissions to access and use specific operations for + a given user when applied to a tenant. Roles are logical groupings of + those permissions to enable common permissions to be easily grouped and + bound to users associated with a given tenant. + +Service Catalog +--------------- + +Keystone also provides a list of REST API endpoints as a definitive list for +an OpenStack installation. Key concepts include: + +Service + An OpenStack service such as nova, swift, glance, or keystone. A service + may have one of more endpoints through which users can interact with + OpenStack services and resources. + +Endpoint + A network accessible address (typically a URL) that represents the API + interface to an OpenStack service. Endpoints may also be grouped into + templates which represent a group of consumable OpenStack services + available across regions. + +Template + A collection of endpoints representing a set of consumable OpenStack + service endpoints. + +Components of Keystone +---------------------- + +Keystone includes a command-line interface which interacts with the Keystone +API for administrating keystone and related services. + +* keystone - runs both keystone-admin and keystone-service +* keystone-admin - the administrative API for manipulating keystone +* keystone-service - the user oriented API for authentication +* keystone-manage - the command line interface to manipulate keystone + +Keystone also includes WSGI middelware to provide authentication support +for Nova and Swift. + +Keystone uses a built-in SQLite datastore - and may use an external LDAP +service to authenticate users instead of using stored credentials. diff --git a/docs/source/backends.rst b/docs/source/backends.rst new file mode 100644 index 0000000000..b3fc2d9194 --- /dev/null +++ b/docs/source/backends.rst @@ -0,0 +1,188 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +======== +Backends +======== + +Keystone supports multiple types of data stores for things like users, tenants, and +tokens, including SQL, LDAP, and memcache. + +SQL +=== + +In the default backend configuration (SQL-only), Keystone depends on the following database tables. + +``users`` +--------- + +``id`` + Auto-incremented primary key. +``name`` + Unqiue username used for authentication via ``passwordCredentials``. +``password`` + Password used for authentication via ``passwordCredentials``. + + Salted and hashed using ``passlib``. +``email`` + Email address (uniqueness is expected, but not enforced). +``enabled`` + If false, the user is unable to authenticate and the user's tokens will fail validation. +``tenant_id`` + Default tenant for the user. + +``tokens`` +---------- + +``id`` + The actual token provided after successful authentication (*plaintext*). +``user_id`` + References the user who owns the token. +``tenant_id`` + (*optional*) References the tenant the token is scoped to. +``expires`` + Indicates the expiration date of the token, after which the token can no longer be validated successfully. + +``tenants`` +----------- + +``id`` + Auto-incremented primary key. +``name`` + Unique string identifying the tenant. +``desc`` + Description of the tenant. +``enabled`` + If false, users are unable to scope to the tenant. + +``roles`` +--------- + +``id`` + Auto-incremented primary key. +``name`` + Name of the role. + + If the role is owned by a service, the role name **must** follow the convention:: + + serviceName:roleName +``desc`` + Description of the role. +``service_id`` + (*optional*) References the service that owns the role. + +``user_roles`` +-------------- + +Maps users to the roles that have been granted to them (*optionally*, within the scope of a tenant). + +``id`` + Auto-incremented primary key. +``user_id`` + References the user the role is granted to. +``role_id`` + References the granted role. +``tenant_id`` + (*optional*) References a tenant upon which this grant is applies. + +``services`` +------------ + +``id`` + Auto-incremented primary key. +``name`` + Unique name of the service. +``type`` + Indicates the type of service (e.g. ``compute``, ``object``, ``identity``, etc). + + This can also be extended to support non-core services. Extended services + follow the naming convention ``extension:type`` (e.g. ``dnsextension:dns``). +``desc`` + Describes the service. +``owner_id`` + (*optional*) References the user who owns the service. + +``credentials`` +--------------- + +Currently only used for Amazon EC2 credential storage, this table is designed to support multiple +types of credentials in the future. + +``id`` + Auto-incremented primary key. +``user_id`` + References the user who owns the credential. +``tenant_id`` + References the tenant upon which the credential is valid. +``types`` + Indicates the type of credential (e.g. ``Password``, ``APIKey``, ``EC2``). +``key`` + Amazon EC2 access key. +``secret`` + Amazon EC2 secret key. + +``endpoints`` +------------- + +Tenant-specific endpoints map endpoint templates to specific tenants. +The ``tenant_id`` which appears here replaces the +``%tenant_id%`` template variable in the specified endpoint template. + +``id`` + Auto-incremented primary key. +``tenant_id`` + References the tenant this endpoint applies to. +``endpoint_template_id`` + The endpoint template to appear in the user's service catalog. + +``endpoint_templates`` +---------------------- + +A multi-purpose model for the service catalog which can be: + +- Provided to users of a specific tenants via ``endpoints``, when ``is_global`` is false. +- Provided to all users as-is, when ``is_global`` is true. + +``id`` + Auto-incremented primary key. +``region`` + Identifies the geographic region the endpoint is physically located within. +``service_id`` + TODO: References the service which owns the endpoints? +``public_url`` + Appears in the service catalog [#first]_. + + Represents an endpoint available on the public Internet. +``admin_url`` + Appears in the service catalog [#first]_. + + Users of this endpoint must have an Admin or ServiceAdmin role. +``internal_url`` + Appears in the service catalog [#first]_. + + Represents an endpoint on an internal, unmetered network. +``enabled`` + If false, this endpoint template will not appear in the service catalog. +``is_global`` + If true, this endpoint can not be mapped to tenant-specific endpoints, and ``%tenant_id%`` will not be substituted in endpoint URL's. Additionally, this endpoint will appear for all users. +``version_id`` + Identifies the version of the API contract that endpoint supports. +``version_list`` + A URL which lists versions supported by the endpoint. +``version_info`` + A URL which provides detailed version info regarding the service. + +.. [#first] ``%tenant_id%`` may be replaced by actual tenant references, depending on the value of ``is_global`` and the existence of a corresponding ``endpoints`` record. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000000..a98d92f88c --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,100 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +==================== +Configuring Keystone +==================== + +.. toctree:: + :maxdepth: 1 + + keystone.conf + man/keystone-manage + +Once Keystone is installed, there are a number of configuration options +available and potentially some initial data to create and set up. + +Sample data / Quick Setup +========================= + +Default sampledata is provided for easy setup and testing in bin/sampeldata. To +set up the sample data run the following command while Keystone is running:: + + $ ./bin/sampledata + +The sample data created comes from the file :doc:`sourcecode/keystone.test.sampledata` + + +Keystone Configuration File +=========================== + +Most configuration is done via configuration files. The default files are +in ``/etc/keystone.conf`` + +When starting up a Keystone server, you can specify the configuration file to +use (see :doc:`controllingservers`). +If you do **not** specify a configuration file, keystone will look in the following +directories for a configuration file, in order: + +* ``~/.keystone`` +* ``~/`` +* ``/etc/keystone`` +* ``/etc`` + +The keystone configuration file should be named ``keystone.conf``. +If you installed keystone via your operating system's +package management system, it is likely that you will have sample +configuration files installed in ``/etc/keystone``. + +In addition to this documentation page, you can check the +``etc/keystone.conf`` sample configuration +files distributed with keystone for example configuration files for each server +application with detailed comments on what each options does. + +Sample Configuration Files +-------------------------- + +Keystone ships with sample configuration files in keystone/etc. These files are: + +1. keystone.conf + + A standard configuration file for running keystone in stand-alone mode. + It has a set of default extensions loaded to support administering Keystone + over REST. It uses a local SQLite database. + +2. memcache.conf + + A configuration that uses memcached for storing tokens (but still SQLite for all + other entities). This requires memcached running. + +3. ssl.conf + + A configuration that runs Keystone with SSL (so all URLs are accessed over HTTPS). + +To run any of these configurations, use the `-c` option:: + + ./keystone -c ../etc/ssl.conf + + + +Usefule Links +------------- + +For a sample configuration file with explanations of the settings, see :doc:`keystone.conf` + +For configuring an LDAP backend, see http://mirantis.blogspot.com/2011/08/ldap-identity-store-for-openstack.html + +For configuration settings of middleware components, see :doc:`middleware` \ No newline at end of file diff --git a/docs/source/configuringservices.rst b/docs/source/configuringservices.rst new file mode 100644 index 0000000000..083c3ec5ed --- /dev/null +++ b/docs/source/configuringservices.rst @@ -0,0 +1,333 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +========================================== +Configuring Services to work with Keystone +========================================== + +.. toctree:: + :maxdepth: 1 + +Once Keystone is installed and running, services need to be configured to work +with it. These are the steps to configure a service to work with Keystone: + +1. Create or get credentials for the service to use + + A set of credentials are needed for each service (they may be + shared if you chose to). Depending on the service, these credentials are + either a username and password or a long-lived token.. + +2. Register the service, endpoints, roles and other entities + + In order for a service to have it's endpoints and roles show in the service + catalog returned by Keystone, a service record needs to be added for the + service. Endpoints and roles associated with that service can then be created. + + This can be done through the REST interface (using the OS-KSCATALOG extension) + or using keystone-manage. + +3. Install and configure middleware for the service to handle authentication + + Clients making calls to the service will pass in an authentication token. The + Keystone middleware will look for and validate that token, taking the + appropriate action. It will also retrive additional information from the token + such as user name, id, tenant name, id, roles, etc... + + The middleware will pass those data down to the service as headers. The + detailed description of this architecture is available here :doc:`middleware_architecture` + +Setting up credentials +====================== + +First admin user - bootstrapping +-------------------------------- + +For a default installation of Keystone, before you can use the REST API, you +need to create your first initial user and grant that user the right to +administer Keystone. + +For the keystone service itself, two +Roles are pre-defined in the keystone configuration file +(:doc:`keystone.conf`). + + #Role that allows admin operations (access to all operations) + keystone-admin-role = Admin + + #Role that allows acting as service (validate tokens, register service, + etc...) + keystone-service-admin-role = KeystoneServiceAdmin + +In order to create your first user, once Keystone is running use +the `keystone-manage` command: + + $ keystone-manage user add admin secrete + $ keystone-manage role add Admin + $ keystone-manage role add KeystoneServiceAdmin + $ keystone-manage role grant Admin admin + $ keystone-manage role grant KeystoneServiceAdmin admin + +This creates the `admin` user (with a password of `secrete`), creates +two roles (`Admin` and `KeystoneServiceAdmin`), and assigns those roles to +the `admin` user. From here, you should now have the choice of using the +administrative API (as well as the :doc:`man/keystone-manage` commands) to +further configure keystone. There are a number of examples of how to use +that API at :doc:`adminAPI_curl_examples`. + + +Setting up services +=================== + +Defining Services and Service Endpoints +--------------------------------------- + +Keystone also acts as a service catalog to let other OpenStack systems know +where relevant API endpoints exist for OpenStack Services. The OpenStack +Dashboard, in particular, uses this heavily - and this **must** be configured +for the OpenStack Dashboard to properly function. + +Here's how we define the services:: + + $ keystone-manage service add nova compute "Nova Compute Service" + $ keystone-manage service add glance image "Glance Image Service" + $ keystone-manage service add swift storage "Swift Object Storage Service" + $ keystone-manage service add keystone identity "Keystone Identity Service" + +Once the services are defined, we create endpoints for them. Each service +has three relevant URL's associated with it that are used in the command: + +* the public API URL +* an administrative API URL +* an internal URL + +The "internal URL" is an endpoint the generally offers the same API as the +public URL, but over a high-bandwidth, low-latency, unmetered (free) network. +You would use that to transfer images from nova to glance for example, and +not the Public URL which would go over the internet and be potentially chargeable. + +The "admin URL" is for administering the services and is not exposed or accessible +to customers without the apporpriate privileges. + +An example of setting up the endpoint for Nova:: + + $ keystone-manage endpointTemplates add RegionOne nova \ + http://nova-api.mydomain:8774/v1.1/%tenant_id% \ + http://nova-api.mydomain:8774/v1.1/%tenant_id% \ + http://nova-api.mydomain:8774/v1.1/%tenant_id% \ + 1 1 + +Glance:: + + $ keystone-manage endpointTemplates add RegionOne glance \ + http://glance.mydomain:9292/v1 \ + http://glance.mydomain:9292/v1 \ + http://glance.mydomain:9292/v1 \ + 1 1 + +Swift:: + + $ keystone-manage endpointTemplates add RegionOne swift \ + http://swift.mydomain:8080/v1/AUTH_%tenant_id% \ + http://swift.mydomain:8080/v1.0/ \ + http://swift.mydomain:8080/v1/AUTH_%tenant_id% \ + 1 1 + +And setting up an endpoint for Keystone:: + + $ keystone-manage endpointTemplates add RegionOne keystone \ + http://keystone.mydomain:5000/v2.0 \ + http://keystone.mydomain:35357/v2.0 \ + http://keystone.mydomain:5000/v2.0 \ + 1 1 + + +Defining an Administrative Service Token +---------------------------------------- + +An Administrative Service Token is a bit of arbitrary text which is configured +in Keystone and used (typically configured into) Nova, Swift, Glance, and any +other OpenStack projects, to be able to use Keystone services. + +This token is an arbitrary text string, but must be identical between Keystone +and the services using Keystone. This token is bound to a user and tenant as +well, so those also need to be created prior to setting it up. + +The *admin* user was set up above, but we haven't created a tenant for that +user yet:: + + $ keystone-manage tenant add admin + +and while we're here, let's grant the admin user the 'Admin' role to the +'admin' tenant:: + + $ keystone-manage role add Admin + $ keystone-manage role grant Admin admin admin + +Now we can create a service token:: + + $ keystone-manage token add 999888777666 admin admin 2015-02-05T00:00 + +This creates a service token of '999888777666' associated to the admin user, +admin tenant, and expires on February 5th, 2015. This token will be used when +configuring Nova, Glance, or other OpenStack services. + +Securing Communications with SSL +-------------------------------- + +To encrypt traffic between services and Keystone, see :doc:`ssl` + + +Setting up OpenStack users +========================== + +Creating Tenants, Users, and Roles +---------------------------------- + +Let's set up a 'demo' tenant:: + + $ keystone-manage tenant add demo + +And add a 'demo' user with the password 'guest':: + + $ keystone-manage user add demo guest + +Now let's add a role of "Member" and grant 'demo' user that role +as it pertains to the tenant 'demo':: + + $ keystone-manage role add Member + $ keystone-manage role grant Member demo demo + +Let's also add the admin user as an Admin role to the demo tenant:: + + $ keystone-manage role grant Admin admin demo + +Creating EC2 credentials +------------------------ + +To add EC2 credentials for the `admin` and `demo` accounts:: + + $ keystone-manage credentials add admin EC2 'admin' 'secretpassword' + $ keystone-manage credentials add admin EC2 'demo' 'secretpassword' + +If you have a large number of credentials to create, you can put them all +into a single large file and import them using :doc:`man/keystone-import`. The +format of the document looks like:: + + credentials add admin EC2 'username' 'password' + credentials add admin EC2 'username' 'password' + +Then use:: + + $ keystone-import `filename` + + +Setting Up Middleware +===================== + +Keystone Auth-Token Middleware +-------------------------------- + +The Keystone auth_token middleware is a WSGI component that can be inserted in +the WSGI pipeline to handle authenticating tokens with Keystone. See :doc:`middleware` +for details on middleware and configuration parameters. + + +Configuring Nova to use Keystone +-------------------------------- + +To configure Nova to use Keystone for authentication, the Nova API service +can be run against the api-paste file provided by Keystone. This is most +easily accomplished by setting the `--api_paste_config` flag in nova.conf to +point to `examples/paste/nova-api-paste.ini` from Keystone. This paste file +included references to the WSGI authentication middleware provided with the +keystone installation. + +When configuring Nova, it is important to create a admin service token for +the service (from the Configuration step above) and include that as the key +'admin_token' in the nova-api-paste.ini. See the documented +:doc:`nova-api-paste` file for references. + +Configuring Swift to use Keystone +--------------------------------- + +Similar to Nova, swift can be configured to use Keystone for authentication +rather than it's built in 'tempauth'. + +1. Add a service endpoint for Swift to Keystone + +2. Configure the paste file for swift-proxy (`/etc/swift/swift-proxy.conf`) + +3. Reconfigure Swift's proxy server to use Keystone instead of TempAuth. + Here's an example `/etc/swift/proxy-server.conf`:: + + [DEFAULT] + bind_port = 8888 + user = + + [pipeline:main] + pipeline = catch_errors cache keystone proxy-server + + [app:proxy-server] + use = egg:swift#proxy + account_autocreate = true + + [filter:keystone] + use = egg:keystone#tokenauth + auth_protocol = http + auth_host = 127.0.0.1 + auth_port = 35357 + admin_token = 999888777666 + delay_auth_decision = 0 + service_protocol = http + service_host = 127.0.0.1 + service_port = 8100 + service_pass = dTpw + cache = swift.cache + + [filter:cache] + use = egg:swift#memcache + set log_name = cache + + [filter:catch_errors] + use = egg:swift#catch_errors + + Note that the optional "cache" property in the keystone filter allows any + service (not just Swift) to register its memcache client in the WSGI + environment. If such a cache exists, Keystone middleware will utilize it + to store validated token information, which could result in better overall + performance. + +4. Restart swift + +5. Verify that keystone is providing authentication to Swift + +Use `swift` to check everything works (note: you currently have to create a +container or upload something as your first action to have the account +created; there's a Swift bug to be fixed soon):: + + $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete post container + $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete stat -v + StorageURL: http://127.0.0.1:8888/v1/AUTH_1234 + Auth Token: 74ce1b05-e839-43b7-bd76-85ef178726c3 + Account: AUTH_1234 + Containers: 1 + Objects: 0 + Bytes: 0 + Accept-Ranges: bytes + X-Trans-Id: tx25c1a6969d8f4372b63912f411de3c3b + +.. WARNING:: + Keystone currently allows any valid token to do anything with any account. + diff --git a/docs/source/controllingservers.rst b/docs/source/controllingservers.rst new file mode 100644 index 0000000000..ba8bfc065b --- /dev/null +++ b/docs/source/controllingservers.rst @@ -0,0 +1,288 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================ +Controlling Keystone Servers +============================ + +This section describes the ways to start, stop, and reload the Keystone +services. + +Keystone Services +----------------- + +Keystone can serve a number of REST APIs and extensions on different TCP/IP +ports. + +The Service API +~~~~~~~~~~~~~~~~ + +The core Keystone +API is primarily a read-only API (the only write operation being POST /tokens +which authenticates a client, and returns a generated token). +This API is sufficient to use OpenStack if all users, roles, endpoints already +exist. This is often the case if Keystone is using an enterprise backend +and the backend is managed through other entperrise tools and business +processes. This core API is called the Service API and can be started +separately from the more complete Admin API. By default, Keystone runs +this API on port 5000. This is not an IANA assigned port and should not +be relied upon (instead, use the Admin API on port 35357 to look for +this endpoint - more on this later) + +The Service API is started using this command in the /bin directory:: + + $ ./keystone-auth + +The Admin API +~~~~~~~~~~~~~ + +Inn order for Keystone to be a fully functional service out of the box, +API extensions that provide full CRUD operations is included with Keystone. +This full set of API calls includes the OS-KSCATALOG, OS-KSADM, and OS-KSEC2 +extensions. These extensions provide a full set of create, read, update, delete +(CRUD) operations that can be used to manage Keystone objects through REST +calls. By default Keystone runs this full REST API on TCP/IP port 35357 +(assigned by IANA to Keystone). + +The Admin API is started using this command in the /bin directory:: + + $ ./keystone-admin + + +Both APIs can be loaded simultaneously (on different ports) using this command:: + + $ ./keystone + +Starting a server +----------------- + +There are two ways to start a Keystone service (either the Service API server +or the Admin API server): + +- Manually calling the server program +- Using the ``keystone-control`` server daemon wrapper program + +We recommend using the second way in production and the first for development +and debugging. + +Manually starting the server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The first is by directly calling the server program, passing in command-line +options and a single argument for a ``paste.deploy`` configuration file to +use when configuring the server application. + +.. note:: + + Keystone ships with an ``etc/`` directory that contains a sample ``paste.deploy`` + configuration files that you can copy to a standard configuration directory and + adapt for your own uses. + +If you do `not` specify a configuration file on the command line, Keystone will +do its best to locate a configuration file in one of the +following directories, stopping at the first config file it finds: + +- ``$CWD`` +- ``~/.keystone`` +- ``~/`` +- ``/etc/keystone`` +- ``/etc`` + +The filename that is searched for is ``keystone.conf`` by default. + +If no configuration file is found, you will see an error, like:: + + $ keystone + ERROR: Unable to locate any configuration file. Cannot load application keystone + +Here is an example showing how you can manually start the ``keystone-auth`` server and ``keystone-registry`` in a shell:: + + $ ./keystone -d + keystone-legacy-auth: INFO ************************************************** + keystone-legacy-auth: INFO Configuration options gathered from config file: + keystone-legacy-auth: INFO /Users/ziadsawalha/Documents/Code/keystone/etc/keystone.conf + keystone-legacy-auth: INFO ================================================ + keystone-legacy-auth: INFO admin_host 0.0.0.0 + keystone-legacy-auth: INFO admin_port 35357 + keystone-legacy-auth: INFO admin_ssl False + keystone-legacy-auth: INFO backends keystone.backends.sqlalchemy + keystone-legacy-auth: INFO ca_certs /etc/keystone/ssl/certs/ca.pem + keystone-legacy-auth: INFO cert_required True + keystone-legacy-auth: INFO certfile /etc/keystone/ssl/certs/keystone.pem + keystone-legacy-auth: INFO debug True + keystone-legacy-auth: INFO default_store sqlite + keystone-legacy-auth: INFO extensions osksadm,oskscatalog,hpidm + keystone-legacy-auth: INFO hash-password True + keystone-legacy-auth: INFO keyfile /etc/keystone/ssl/private/keystonekey.pem + keystone-legacy-auth: INFO keystone-admin-role Admin + keystone-legacy-auth: INFO keystone-service-admin-role KeystoneServiceAdmin + keystone-legacy-auth: INFO log_dir . + keystone-legacy-auth: INFO log_file keystone.log + keystone-legacy-auth: INFO service-header-mappings { + 'nova' : 'X-Server-Management-Url', + 'swift' : 'X-Storage-Url', + 'cdn' : 'X-CDN-Management-Url'} + keystone-legacy-auth: INFO service_host 0.0.0.0 + keystone-legacy-auth: INFO service_port 5000 + keystone-legacy-auth: INFO service_ssl False + keystone-legacy-auth: INFO verbose False + keystone-legacy-auth: INFO ************************************************** + passlib.registry: INFO registered crypt handler 'sha512_crypt': + Starting the RAX-KEY extension + Starting the Legacy Authentication component + admin : INFO ************************************************** + admin : INFO Configuration options gathered from config file: + admin : INFO /Users/ziadsawalha/Documents/Code/keystone/etc/keystone.conf + admin : INFO ================================================ + admin : INFO admin_host 0.0.0.0 + admin : INFO admin_port 35357 + admin : INFO admin_ssl False + admin : INFO backends keystone.backends.sqlalchemy + admin : INFO ca_certs /etc/keystone/ssl/certs/ca.pem + admin : INFO cert_required True + admin : INFO certfile /etc/keystone/ssl/certs/keystone.pem + admin : INFO debug True + admin : INFO default_store sqlite + admin : INFO extensions osksadm,oskscatalog,hpidm + admin : INFO hash-password True + admin : INFO keyfile /etc/keystone/ssl/private/keystonekey.pem + admin : INFO keystone-admin-role Admin + admin : INFO keystone-service-admin-role KeystoneServiceAdmin + admin : INFO log_dir . + admin : INFO log_file keystone.log + admin : INFO service-header-mappings { + 'nova' : 'X-Server-Management-Url', + 'swift' : 'X-Storage-Url', + 'cdn' : 'X-CDN-Management-Url'} + admin : INFO service_host 0.0.0.0 + admin : INFO service_port 5000 + admin : INFO service_ssl False + admin : INFO verbose False + admin : INFO ************************************************** + Using config file: /Users/ziadsawalha/Documents/Code/keystone/etc/keystone.conf + Service API (ssl=False) listening on 0.0.0.0:5000 + Admin API (ssl=False) listening on 0.0.0.0:35357 + eventlet.wsgi.server: DEBUG (77128) wsgi starting up on http://0.0.0.0:5000/ + eventlet.wsgi.server: DEBUG (77128) wsgi starting up on http://0.0.0.0:35357/ + + $ sudo keystone-registry keystone-registry.conf & + jsuh@mc-ats1:~$ 2011-04-13 14:51:16 INFO [sqlalchemy.engine.base.Engine.0x...feac] PRAGMA table_info("images") + 2011-04-13 14:51:16 INFO [sqlalchemy.engine.base.Engine.0x...feac] () + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Col ('cid', 'name', 'type', 'notnull', 'dflt_value', 'pk') + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (0, u'created_at', u'DATETIME', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (1, u'updated_at', u'DATETIME', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (2, u'deleted_at', u'DATETIME', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (3, u'deleted', u'BOOLEAN', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (4, u'id', u'INTEGER', 1, None, 1) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (5, u'name', u'VARCHAR(255)', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (6, u'disk_format', u'VARCHAR(20)', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (7, u'container_format', u'VARCHAR(20)', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (8, u'size', u'INTEGER', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (9, u'status', u'VARCHAR(30)', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (10, u'is_public', u'BOOLEAN', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (11, u'location', u'TEXT', 0, None, 0) + 2011-04-13 14:51:16 INFO [sqlalchemy.engine.base.Engine.0x...feac] PRAGMA table_info("image_properties") + 2011-04-13 14:51:16 INFO [sqlalchemy.engine.base.Engine.0x...feac] () + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Col ('cid', 'name', 'type', 'notnull', 'dflt_value', 'pk') + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (0, u'created_at', u'DATETIME', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (1, u'updated_at', u'DATETIME', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (2, u'deleted_at', u'DATETIME', 0, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (3, u'deleted', u'BOOLEAN', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (4, u'id', u'INTEGER', 1, None, 1) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (5, u'image_id', u'INTEGER', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (6, u'key', u'VARCHAR(255)', 1, None, 0) + 2011-04-13 14:51:16 DEBUG [sqlalchemy.engine.base.Engine.0x...feac] Row (7, u'value', u'TEXT', 0, None, 0) + + $ ps aux | grep keystone + myuser 77148 0.0 0.0 2434892 472 s012 U+ 11:50AM 0:00.01 grep keystone + myuser 77128 0.0 0.6 2459356 25360 s011 S+ 11:48AM 0:00.82 python ./keystone -d + +Simply supply the configuration file as the first argument +and then any common options +you want to use (``-d`` was used above to show some of the debugging +output that the server shows when starting up. Call the server program +with ``--help`` to see all available options you can specify on the +command line.) + +Using ``--trace-calls`` is useful for showing a trace of calls (errors in red) +for debugging. + +For more information on configuring the server via the ``paste.deploy`` +configuration files, see the section entitled +:doc:`Configuring Keystone ` + +Note that the server `daemonizes` itself by using the standard +shell backgrounding indicator, ``&``, in the previous example. For most use cases, we recommend +using the ``keystone-control`` server daemon wrapper for daemonizing. See below +for more details on daemonization with ``keystone-control``. + +Using ``keystone-control`` to start the server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The second way to start up a Keystone server is to use the ``keystone-control`` +program. ``keystone-control`` is a wrapper script that allows the user to +start, stop, restart, and reload the other Keystone server programs in +a fashion that is more conducive to automation and scripting. + +Servers started via the ``keystone-control`` program are always `daemonized`, +meaning that the server program process runs in the background. + +To start a Keystone server with ``keystone-control``, simply call +``keystone-control`` with a server and the word "start", followed by +any command-line options you wish to provide. Start the server with ``keystone-control`` +in the following way:: + + $ sudo keystone-control start [CONFPATH] + +.. note:: + + You must use the ``sudo`` program to run ``keystone-control`` currently, as the + pid files for the server programs are written to /var/run/keystone/ + +Start the ``keystone-admin`` server using ``keystone-control``:: + + $ sudo keystone-control admin start + Starting keystone-admin with /etc/keystone.conf + +The same ``paste.deploy`` configuration files are used by ``keystone-control`` +to start the Keystone server programs, and you can specify (as the example above +shows) a configuration file when starting the server. + +Stopping a server +----------------- + +If you started a Keystone server manually and did not use the ``&`` backgrounding +function, simply send a terminate signal to the server process by typing +``Ctrl-C`` + +If you started the Keystone server using ``keystone-control``, you can +use the ``keystone-control`` program to stop it:: + + $ sudo keystone-control stop + +For example:: + + $ sudo keystone-control auth stop + Stopping keystone-auth pid: 77401 signal: 15 + +Restarting a server +------------------- + +Restart the Keystone server using ``keystone-control``:: + + $ sudo keystone-control admin restart /etc/keystone.conf + Stopping keystone-admin pid: 77401 signal: 15 + Starting keystone-admin with /etc/keystone.conf diff --git a/docs/source/endpoints.rst b/docs/source/endpoints.rst new file mode 100644 index 0000000000..84a42e091f --- /dev/null +++ b/docs/source/endpoints.rst @@ -0,0 +1,430 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +================================ +Endpoints and Endpoint Templates +================================ + +.. toctree:: + :maxdepth: 1 + +What are Endpoints? +------------------- + +Simply, endpoints are URLs that point to OpenStack services. When you +authenticate to Keystone you get back a token which has a service catalog in +it. The service catalog is basically a list of the OpenStack services that +you have access to and the URLs you can use to get to them; their endpoints. + +Here is an example response from Keystone when you authenticate:: + + { + "access":{ + "token":{ + "id":"ab48a9efdfedb23ty3494", + "expires":"2010-11-01T03:32:15-05:00", + "tenant":{ + "id": "t1000", + "name": "My Project" + } + }, + "user":{ + "id":"u123", + "name":"jqsmith", + "roles":[{ + "id":"100", + "name":"compute:admin" + }, + { + "id":"101", + "name":"object-store:admin", + "tenantId":"t1000" + } + ], + "roles_links":[] + }, + "serviceCatalog":[{ + "name":"Nova", + "type":"compute", + "endpoints":[{ + "tenantId":"t1000", + "publicURL":"https://compute.north.host.com/v1/t1000", + "internalURL":"https://compute.north.internal/v1/t1000", + "region":"North", + "versionId":"1", + "versionInfo":"https://compute.north.host.com/v1/", + "versionList":"https://compute.north.host.com/" + }, + { + "tenantId":"t1000", + "publicURL":"https://compute.north.host.com/v1.1/t1000", + "internalURL":"https://compute.north.internal/v1.1/t1000", + "region":"North", + "versionId":"1.1", + "versionInfo":"https://compute.north.host.com/v1.1/", + "versionList":"https://compute.north.host.com/" + } + ], + "endpoints_links":[] + }, + { + "name":"Swift", + "type":"object-store", + "endpoints":[{ + "tenantId":"t1000", + "publicURL":"https://storage.north.host.com/v1/t1000", + "internalURL":"https://storage.north.internal/v1/t1000", + "region":"North", + "versionId":"1", + "versionInfo":"https://storage.north.host.com/v1/", + "versionList":"https://storage.north.host.com/" + }, + { + "tenantId":"t1000", + "publicURL":"https://storage.south.host.com/v1/t1000", + "internalURL":"https://storage.south.internal/v1/t1000", + "region":"South", + "versionId":"1", + "versionInfo":"https://storage.south.host.com/v1/", + "versionList":"https://storage.south.host.com/" + } + ] + }, + { + "name":"DNS-as-a-Service", + "type":"dnsextension:dns", + "endpoints":[{ + "tenantId":"t1000", + "publicURL":"https://dns.host.com/v2.0/t1000", + "versionId":"2.0", + "versionInfo":"https://dns.host.com/v2.0/", + "versionList":"https://dns.host.com/" + } + ] + } + ] + } + } + +Note the following about this response: + +#. There are two endpoints given to the Nova compute service. The only + difference between them is the version (1.0 vs. 1.1). This allows for code + written to look for the version 1.0 endpoint to still work even after the 1.1 + version is released. + +#. There are two endpoints for the Swift object-store service. The difference + between them is they are in different regions (North and South). + +#. Note the DNS service is global; it does not have a Region. Also, since DNS + is not a core OpenStack service, the endpoint type is "dnsextension:dns" + showing it is coming from an extension to the Keystone service. + +#. The Region, Tenant, and versionId are listed under the endpoint. You do not + (and should not) have to parse those out of the URL. In fact, they may not be + embedded in the URL if the service developer so chooses. + + +What do the fields in an Endpoint mean? +--------------------------------------- + +The schema definition for an endpoint is in endpoints.xsd under +keystone/content/common/xsd in the Keystone code repo. The fields are: + +id + A unique ID for the endpoint. + +type + The OpenStack-registered type (ex. 'compute', 'object-store', 'image service') + This can also be extended using the OpenStack Extension mechanism to support + non-core services. Extended services will be in the form ``extension:type`` + (e.g. ``dnsextension:dns``) + +name + This can be anything that the operator of OpenStack chooses. It could be a + brand or marketing name (ex. Rackspace Cloud Servers). + +region + This is a string that identifies the region where this endpoint exists. + Examples are 'North America', 'Europe', 'Asia'. Or 'North' and 'South'. Or + 'Data Center 1', 'Data Center 2'. + The list of regions and what a region means is decided by the operator. The + spec treats them as opaque strings. + +publicURL + This is the URL to use to access that endpoint over the internet. + +internalURL + This is the URL to use to communicate between services. This is genenrally + a way to communicate between services over a high bandwidth, low latency, + unmetered (free, no bandwidth charges) network. An example would be if you + want to access a swift cluster from inside your Nova VMs and want to make + sure the communication stays local and does not go over a public network + and rack up your bandwidth charges. + +adminURL + This is the URL to use to administer the service. In Keystone, this URL + is only shown to users with the appropriate rights. + +tenantId + If an endpoint is specific to a tenant, the tenantId field identifies the + tenant that URL applies to. Some operators include the tenant in the + URLs for a service, while others may provide one endpoint and use some + other mechanism to identify the tenant. This field is therefore optional. + Having this field also means you do not have to parse the URL to identify + a tenant if the operator includes it in the URL. + +versionId + This identifies the version of the API contract that endpoint supports. + While many APIs include the version in the URL (ex: https://compute.host/v1), + this field allows you to identify the version without parsing the URL. It + therefore also allows operators and service developers to publish endpoints + that do not have versions embedded in the URL. + +versionInfo + This is the URL to call to get some information on the version. This returns + information in this format:: + + { + "version": { + "id": "v2.0", + "status": "CURRENT", + "updated": "2011-01-21T11:33:21-06:00", + "links": [ + { + "rel": "self", + "href": "http://identity.api.openstack.org/v2.0/" + }, { + "rel": "describedby", + "type": "application/pdf", + "href": "http://docs.openstack.org/identity/api/v2.0/identity-latest.pdf" + }, { + "rel": "describedby", + "type": "application/vnd.sun.wadl+xml", + "href": "http://docs.openstack.org/identity/api/v2.0/identity.wadl" + } + ], + "media-types": [ + { + "base": "application/xml", + "type": "application/vnd.openstack.identity+xml;version=2.0" + }, { + "base": "application/json", + "type": "application/vnd.openstack.identity+json;version=2.0" + } + ] + } + } + +versionList + + This is the URL to call to find out which versions are supported at that + endpoint. The response is in this format:: + + { + "versions":[{ + "id":"v1.0", + "status":"DEPRECATED", + "updated":"2009-10-09T11:30:00Z", + "links":[{ + "rel":"self", + "href":"http://identity.api.openstack.org/v1.0/" + } + ] + }, + { + "id":"v1.1", + "status":"CURRENT", + "updated":"2010-12-12T18:30:02.25Z", + "links":[{ + "rel":"self", + "href":"http://identity.api.openstack.org/v1.1/" + } + ] + }, + { + "id":"v2.0", + "status":"BETA", + "updated":"2011-05-27T20:22:02.25Z", + "links":[{ + "rel":"self", + "href":"http://identity.api.openstack.org/v2.0/" + } + ] + } + ], + "versions_links":[] + } + + Here, the response shows that the endpoint supports version 1.0, 1.1, and 2.0. + It also shows that 1.0 is in DEPRECTAED status and 2.0 is in BETA. + +What are Endpoint Templates? +---------------------------- + +Endpoint Templates are a way for an administrator to manage endpoints en masse. +They provide a way to define Endpoints that apply to many or all tenants +without having to a create each endpoint on each tenant manually. Without +Endpoint Templates, if I wanted to create Endpoints for each tenant in my +OpenStack deployment, I'd have to manually create a bunch of endpoints on +each tenant (probably when I created the tenant). And then I'd have to go change +them all whenever a service changed versions or I added a new service. + +To provide a simpler mechanism to manage endpoints on tenants, Keystone uses +Endpoint Templates. I can, for example, define a template with parametrized URLs +and set it's `global` to true and that will show up as an endpoint on all the tenants +I have. Here is an example: + +Define a global Endpoint Template:: + + $ ./keystone-manage endpointTemplates add North nova https://compute.north.example.com/v1/%tenant_id%/ https://compute.north.example.corp/v1/ https://compute.north.example.local/v1/%tenant_id%/ 1 1 + + The arguments are: object_type action 'region' 'service_name' 'publicURL' 'adminURL' 'internalURL' 'enabled' 'global' + +This creates a global endpoint (global means it gets applied to all tenants automatically). + +Now, when a user authenticates, they get that endpoint in their service catalog. Here's an example +authentication request for use against tenant 1:: + + $ curl -H "Content-type: application/json" -d '{"auth":{"passwordCredentials":{"username":"joeuser","password":"secrete"}, "tenantId": "1"}}' http://localhost:5000/v2.0/tokens + +The response is:: + + { + "access": { + "serviceCatalog": [ + { + "endpoints": [ + { + "internalURL": "https://compute.north.example.local", + "publicURL": "https://compute.north.example.com/v1/1/", + "region": "North" + } + ], + "name": "nova", + "type": "compute" + } + ], + "token": { + "expires": "2012-02-05T00:00:00", + "id": "887665443383838", + "tenant": { + "id": "1", + "name": "customer-x" + } + }, + "user": { + "id": "1", + "name": "joeuser", + "roles": [ + { + "id": "3", + "name": "Member", + "tenantId": "1" + } + ] + } + } + } + +Notice the adminURL is not showing (this user is a regular user and does not +have rights to see the adminURL) and the tenant ID has been substituted in the +URL:: + + "publicURL": "https://compute.north.example.com/v1/1/", + +This endpoint will show up for all tenants. The OpenStack administrator does +not need to create the endpoint manually. + +.. note:: Endpoint Templates are not part of the core Keystone API (but Endpoints are). + + +What parameters can I use in a Template URL +------------------------------------------- + +Currently the only parameterization available is %tenant_id% which gets +substituted by the Tenant ID. + + +Endpoint Template Types: Global or not +-------------------------------------- + +When the global flag is set to true on an Endpoint Template, it means it should +be available to all tenants. Whenever someone authenticates to a tenant, they +will see the Endpoint generated by that template. + +When the global flag is not set, the template only shows up when it is added to +a tenant manually. To add an endpoint to a tenant manually, you must create +the Endpoint and supply the Endpoint Template ID: + +Create the Endpoint Template:: + + $ ./keystone-manage endpointTemplates add West nova https://compute.west.example.com/v1/%tenant_id%/ https://compute.west.example.corp https://compute.west.example.local 1 0 + + Note the 0 at the end - this Endpoint Template is not global. So it will not show up for users authenticating. + +Find the Endpoint Template ID:: + + $ ./keystone-manage endpointTemplates list + + All EndpointTemplates + id service type region enabled is_global Public URL Admin URL + ------------------------------------------------------------------------------- + 15 nova compute North True True https://compute.north.example.com/v1/%tenant_id%/ https://compute.north.example.corp + 16 nova compute West True False https://compute.west.example.com/v1/%tenant_id%/ https://compute.west.example.corp + +Add the Endpoint to the tenant:: + + $ ./keystone-manage endpoint add customer-x 16 + +Now, when the user authenticates, they get the endpoint:: + + { + "internalURL": "https://compute.west.example.local", + "publicURL": "https://compute.west.example.com/v1/1/", + "region": "West" + } + +Who can see the AdminURL? +------------------------- + +Users who have the Keystone `Admin` or `Service Admin` roles will see the +AdminURL when they authenticate or when they retrieve token information: + +Using an administrator token to authenticate, GET a client token's endpoints:: + + $ curl -H "X-Auth-Token: 999888777666" http://localhost:35357/v2.0/tokens/887665443383838/endpoints + + { + "endpoints": [ + { + "adminURL": "https://compute.west.example.corp", + "id": 6, + "internalURL": "https://compute.west.example.local", + "name": "nova", + "publicURL": "https://compute.west.example.com/v1/1/", + "region": "West", + "tenantId": 1, + "type": "compute" + } + ], + "endpoints_links": [ + { + "href": "http://127.0.0.1:35357/tokens/887665443383838/endpoints?marker=6&limit=10", + "rel": "next" + } + ] + } diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst new file mode 100644 index 0000000000..539bef3943 --- /dev/null +++ b/docs/source/extensions.rst @@ -0,0 +1,183 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +========== +Extensions +========== + +Extensions support adding features and functions to OpenStack APIs at any time, without prior +approval or waiting for a new API and release cycles. + +The extension framework is in development and documented in extensions_ and extensionspresentation_. + +This document describes the extensions included with Keystone, how to enable and disable them, +and briefly touches on how to write your own extensions. + +.. _extensions: http://docs.openstack.org/trunk/openstack-compute/developer/openstack-api-extensions/content/ch02s01.html +.. _extensionspresentation: http://www.slideshare.net/RackerWilliams/openstack-extensions + +Built-in Extensions +------------------- + +Keystone ships with a number of extensions found under the +``keystone/contib/extensions`` folder. + +The following built-in extensions are included: + +OS-KSADM + + This is an extensions that supports managing users, tenants, and roles + through the API. Without this extensions, the ony way to manage those + objects is through keystone-manage or directly in the underlying database. + + This is an Admin API extension only. + +OS-KSCATALOG + + This extensions supports managing Endpoints and prrovides the Endpoint + Template mechanism for managing bulk endpoints. + + This is an Admin API extension only. + +OS-EC2 + + This extension adds support for EC2 credentials. + + This is an Admin and Service API extension. + +RAX-GRP + + This extension adds functionality the enables groups. + + This is an Admin and Service API extension. + +RAX-KEY + + This extensions adds support for authentication with an API Key (the core + Keystone API only supports username/password credentials) + + This is an Admin and Service API extension. + +HP-IDM + + This extension adds capability to filter roles with optional service IDs + for token validation to mitigate security risks with role name conflicts. + See https://bugs.launchpad.net/keystone/+bug/890411 for more details. + + This is an Admin API extension. Applicable to validate token (GET) + and check token (HEAD) APIs only. + +OS-KSVALIDATE + + This extensions supports admin calls to /tokens without having to specify + the token ID in the URL. Instead, the ID is supplied in a header called + X-Subject-Token. This is provided as an alternative to address any security + concerns that arise when token IDs are passed as part of the URL which is + often (and by default) logged to insecure media. + + This is an Admin API extension only. + +.. note:: + + The included extensions are in the process of being rewritten. Currently + osksadm, oskscatalog, hpidm, and osksvalidate work with this new + extensions design. + + +Enabling & Disabling Extensions +------------------------------- + +The Keystone conf file has a property called extensions. This property holds +the list of supported extensions that you want enabled. If you want to +add/remove an extension from being supported, add/remove the extension key +from this property. The key is the name of the folder of the extension +under the keystone/contrib/extensions folder. + +.. note:: + + If you want to load different extensions in the service API than the Admin API + you need to use different config files. + +Creating New Extensions +----------------------- + +#. **Adopt a unique organization abbreviation.** + + This prefix should uniquely identify your organization within the community. + The goal is to avoid schema and resource collisions with similiar extensions. + (e.g. ``OS`` for OpenStack, ``RAX`` for Rackspace, or ``HP`` for Hewlett-Packard) + +#. **Adopt a unique extension abbreviation.** + + Select an abbreviation to identify your extension, and append to + your organization prefix using a hyphen (``-``), by convention + (e.g. ``OS-KSADM`` (for OpenStack's Keystone Administration extension). + + This combination is referred to as your extension's prefix. + +#. **Determine the scope of your extension.** + + Extensions can enhance the Admin API, Service API or both. + +#. **Create a new module.** + + Create a module to isolate your namespace based on the extension prefix + you selected:: + + keystone/contrib/extensions/admin + + ... and/or:: + + keystone/contrib/extensions/service/ + + ... based on which API you are enhancing. + + .. note:: + + In the future, we will support loading external extensions. + +#. Add static extension files for JSON (``*.json``) and XML + (``*.xml``) to the new extension module. + + Refer to `Service Guide `_ + `Sample extension XML `_ + `Sample extension JSON `_ for the the content and structure. + +#. If your extension is adding additional methods override the base class + ``BaseExtensionHandler``, name it ``ExtensionHandler``, and add your methods. + +#. **Document your work.** + + Provide documentation to support your extension. + + Extensions documentation, WADL, and XSD files can be stored in the + ``keystone/content`` folder. + +#. Add your extension name to the list of supported extensions in The + ``keystone.conf`` file. + +Which extensions are enabled? +----------------------------- + +Discover which extensions are available (service API):: + + curl http://localhost:5000/v2.0/extensions + +... or (admin API):: + + curl http://localhost:35357/v2.0/extensions + +The response will list the extensions available. diff --git a/docs/source/images/305.svg b/docs/source/images/305.svg new file mode 100644 index 0000000000..7d79464e2b --- /dev/null +++ b/docs/source/images/305.svg @@ -0,0 +1,158 @@ + + + + + + + + + + image/svg+xml + + + + + + + + Request + service directly + + + Auth + Component + 305 + Use proxy to + redirect to Auth + + + + + + + OpenStack + Service + + + diff --git a/docs/source/images/authComp.svg b/docs/source/images/authComp.svg new file mode 100644 index 0000000000..d344b87102 --- /dev/null +++ b/docs/source/images/authComp.svg @@ -0,0 +1,174 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + Auth + Component + + + OpenStack + Service + + + + + + + Reject + unauthenticated + requests + Forward + authenticated + requests + + + diff --git a/docs/source/images/both.svg b/docs/source/images/both.svg new file mode 100644 index 0000000000..d29872a4a6 --- /dev/null +++ b/docs/source/images/both.svg @@ -0,0 +1,135 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + Auth + Component + + Auth + Component + + + OpenStack + Service + + + + + diff --git a/docs/source/images/graphs_305.svg b/docs/source/images/graphs_305.svg new file mode 100644 index 0000000000..1dff61a686 --- /dev/null +++ b/docs/source/images/graphs_305.svg @@ -0,0 +1,41 @@ + + + + + + +Handle305 + + +AuthComp + +Auth +Component + + +Service + +OpenStack +Service + + + +Service:n->AuthComp:n + + +305 Use Proxy +To Redirect to Auth + + + +Start:sw->Service + + +Request +Service Directly + + + diff --git a/docs/source/images/graphs_authComp.svg b/docs/source/images/graphs_authComp.svg new file mode 100644 index 0000000000..6be629c12e --- /dev/null +++ b/docs/source/images/graphs_authComp.svg @@ -0,0 +1,48 @@ + + + + + + +AuthComp + + +AuthComp + +Auth +Component + + + +AuthComp->Reject + + +Reject +Unauthenticated +Requests + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Forward +Authenticated +Requests + + + +Start->AuthComp + + + + + diff --git a/docs/source/images/graphs_authCompDelegate.svg b/docs/source/images/graphs_authCompDelegate.svg new file mode 100644 index 0000000000..4788829a45 --- /dev/null +++ b/docs/source/images/graphs_authCompDelegate.svg @@ -0,0 +1,53 @@ + + + + + + +AuthCompDelegate + + +AuthComp + +Auth +Component + + + +AuthComp->Reject + + +Reject Requests +Indicated by the Service + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Forward Requests +with Identiy Status + + +Service->AuthComp + + +Send Response OR +Reject Message + + + +Start->AuthComp + + + + + diff --git a/docs/source/images/graphs_both.svg b/docs/source/images/graphs_both.svg new file mode 100644 index 0000000000..6aa8761260 --- /dev/null +++ b/docs/source/images/graphs_both.svg @@ -0,0 +1,36 @@ + + + + + + +Both + + +AuthComp + +Auth +Component + + +Together + + + +Auth +Component + + +OpenStack +Service + + +AuthComp->Together:OStack:n + + + + + diff --git a/docs/source/images/graphs_delegate_accept.svg b/docs/source/images/graphs_delegate_accept.svg new file mode 100644 index 0000000000..1d86cadfc6 --- /dev/null +++ b/docs/source/images/graphs_delegate_accept.svg @@ -0,0 +1,52 @@ + + + + + + +DelegateAcceptAuth + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp->Start + + +200 Okay + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U +X-Identity-Status: Confirmed + + +Service->AuthComp + + +200 Okay + + + diff --git a/docs/source/images/graphs_delegate_forbiden_basic.svg b/docs/source/images/graphs_delegate_forbiden_basic.svg new file mode 100644 index 0000000000..dcd62b775d --- /dev/null +++ b/docs/source/images/graphs_delegate_forbiden_basic.svg @@ -0,0 +1,53 @@ + + + + + + +DelegateRejectForbidden + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp->Start + + +403 Forbidden + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U +X-Identity-Status: Confirmed + + +Service->AuthComp + + +403 Forbidden +WWW-Authenticate: Delegated + + + diff --git a/docs/source/images/graphs_delegate_forbiden_proxy.svg b/docs/source/images/graphs_delegate_forbiden_proxy.svg new file mode 100644 index 0000000000..df53212b40 --- /dev/null +++ b/docs/source/images/graphs_delegate_forbiden_proxy.svg @@ -0,0 +1,52 @@ + + + + + + +DelegateForbiddnProxy + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp->Start + + +500 Internal Error + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U +X-Identity-Status: Confirmed + + +Service->AuthComp + + +403 Forbidden + + + diff --git a/docs/source/images/graphs_delegate_reject_basic.svg b/docs/source/images/graphs_delegate_reject_basic.svg new file mode 100644 index 0000000000..a33ea095f1 --- /dev/null +++ b/docs/source/images/graphs_delegate_reject_basic.svg @@ -0,0 +1,55 @@ + + + + + + +DelegateRejectAuthBasic + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic Yjpw + + +AuthComp->Start + + +401 Unauthorized +WWW-Authenticate: Basic +Realm="API Realm" + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy b +X-Identity-Status: Indeterminate + + +Service->AuthComp + + +401 Unauthorized +WWW-Authenticate: Delegated + + + diff --git a/docs/source/images/graphs_delegate_reject_oauth.svg b/docs/source/images/graphs_delegate_reject_oauth.svg new file mode 100644 index 0000000000..760adeb694 --- /dev/null +++ b/docs/source/images/graphs_delegate_reject_oauth.svg @@ -0,0 +1,56 @@ + + + + + + +DelegateRejectAuthOAuth + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: OAuth 000-999-222 + + +AuthComp->Start + + +401 Unauthorized +WWW-Authenticate: OAuth +Realm=’API Realm’, +Error=’invalid-token’ + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy +X-Identity-Status: Indeterminate + + +Service->AuthComp + + +401 Unauthorized +WWW-Authenticate: Delegated + + + diff --git a/docs/source/images/graphs_delegate_unimplemented.svg b/docs/source/images/graphs_delegate_unimplemented.svg new file mode 100644 index 0000000000..8c4fdc6bf3 --- /dev/null +++ b/docs/source/images/graphs_delegate_unimplemented.svg @@ -0,0 +1,53 @@ + + + + + + +DelegateUnimplemented + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp->Start + + +500 Internal Error + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U +X-Identity-Status: Confirmed + + +Service->AuthComp + + +501 Unimplemented +WWW-Authenticate: Delegated + + + diff --git a/docs/source/images/graphs_mapper.svg b/docs/source/images/graphs_mapper.svg new file mode 100644 index 0000000000..52c6c55b6d --- /dev/null +++ b/docs/source/images/graphs_mapper.svg @@ -0,0 +1,73 @@ + + + + + + +Mapper + + + +Mapper + +Mapper + + +Start->Mapper + + + + +Auths + + + +Auth1 + + +Auth2 + + +Auth3 + + +Mapper:sw->Auths:auth1 + + + + +Mapper:s->Auths:auth2 + + + + +Mapper:se->Auths:auth3 + + + + +Service + +OpenStack +Service + + +Auths:auth1->Service + + + + +Auths:auth2->Service + + + + +Auths:auth3->Service + + + + + diff --git a/docs/source/images/graphs_proxyAuth.svg b/docs/source/images/graphs_proxyAuth.svg new file mode 100644 index 0000000000..7b94b07740 --- /dev/null +++ b/docs/source/images/graphs_proxyAuth.svg @@ -0,0 +1,51 @@ + + + + + + +ProxyAuth + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp:w->Start + + +500 Internal Error + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U + + +Service:w->AuthComp + + +403 Forbidden + + + diff --git a/docs/source/images/graphs_separate.svg b/docs/source/images/graphs_separate.svg new file mode 100644 index 0000000000..376e59880a --- /dev/null +++ b/docs/source/images/graphs_separate.svg @@ -0,0 +1,30 @@ + + + + + + +Seperate + + +AuthComp + +Auth +Component + + +Service + +OpenStack +Service + + +AuthComp->Service + + + + + diff --git a/docs/source/images/graphs_standard_accept.svg b/docs/source/images/graphs_standard_accept.svg new file mode 100644 index 0000000000..bddf4b5f16 --- /dev/null +++ b/docs/source/images/graphs_standard_accept.svg @@ -0,0 +1,51 @@ + + + + + + +StandardAcceptAuth + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic VTpQ + + +AuthComp->Start + + +200 Okay + + +Service + +OpenStack +Service + + +AuthComp->Service + + +Authorization: Basic dTpw +X-Authorization: Proxy U + + +Service->AuthComp + + +200 Okay + + + diff --git a/docs/source/images/graphs_standard_reject.svg b/docs/source/images/graphs_standard_reject.svg new file mode 100644 index 0000000000..6020ad67a5 --- /dev/null +++ b/docs/source/images/graphs_standard_reject.svg @@ -0,0 +1,39 @@ + + + + + + +StandardRejectAuth + + + +AuthComp + +Auth +Component + + +Start->AuthComp + + +Authorization: Basic Yjpw + + +AuthComp->Start + + +401 Unauthorized +WWW-Authenticate: Basic Realm="API Realm" + + +Service + +OpenStack +Service + + + diff --git a/docs/source/images/graphs_together.svg b/docs/source/images/graphs_together.svg new file mode 100644 index 0000000000..1425a28baa --- /dev/null +++ b/docs/source/images/graphs_together.svg @@ -0,0 +1,24 @@ + + + + + + +Together + + +Together + + +Auth +Component + + +OpenStack +Service + + + diff --git a/docs/source/images/images_layouts.svg b/docs/source/images/images_layouts.svg new file mode 100644 index 0000000000..e7fe7a9521 --- /dev/null +++ b/docs/source/images/images_layouts.svg @@ -0,0 +1,200 @@ + + + + + + image/svg+xml + + + + + + + + Auth Layouts + (a) + (b) + + + Together + + + + Together + + + Auth + Component + + + OpenStack + Service + + + + Seperate + + + + AuthComp + + Auth + Component + + + + Service + + OpenStack + Service + + + + AuthComp->Service + + + + + diff --git a/docs/source/images/layouts.svg b/docs/source/images/layouts.svg new file mode 100644 index 0000000000..fdf61b7da7 --- /dev/null +++ b/docs/source/images/layouts.svg @@ -0,0 +1,215 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + Auth + Component + + + OpenStack + Service + + + Option + ( + b + ) + + + Auth + Component + + + OpenStack + Service + Option + ( + a + ) + + + + diff --git a/docs/source/images/mapper.svg b/docs/source/images/mapper.svg new file mode 100644 index 0000000000..b5a2b7b12f --- /dev/null +++ b/docs/source/images/mapper.svg @@ -0,0 +1,237 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + OpenStack + Service + + + + + + + + + + + + + + + Mapper + + + Auth + 1 + + + Auth + 2 + + + Auth + 3 + + + + + diff --git a/docs/source/images/proxyAuth.svg b/docs/source/images/proxyAuth.svg new file mode 100644 index 0000000000..f60b40d813 --- /dev/null +++ b/docs/source/images/proxyAuth.svg @@ -0,0 +1,238 @@ + + + + + + + + + + image/svg+xml + + + + + + + + Authorization + : + Basic dTpw + X + - + Authorization + : + Proxy U + Authorization + : + Basic VTpQ + 500 + Internal Error + 403 + Proxy Unauthorized + + + + + Auth + Component + + + + + OpenStack + Service + + + + + + + diff --git a/docs/source/middleware.rst b/docs/source/middleware.rst new file mode 100644 index 0000000000..69506ee2d6 --- /dev/null +++ b/docs/source/middleware.rst @@ -0,0 +1,169 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +========== +Middleware +========== + +The Keystone middleware sits in front of an OpenStack service and handles authenticating +incoming requests. The middleware was designed according to `this spec`. + +The middleware is found in source under Keystone/middleware. + +The middleware supports two interfaces; WSGI and REST/HTTP. + +.. _`this spec`: http://wiki.openstack.org/openstack-authn + +REST & HTTP API +=============== + +If an unauthenticated call comes in, the middleware will respond with a 401 Unauthorized error. As per +HTTP standards, it will also return a WWW-Authenticate header informing the caller +of what protocols are supported. For Keystone authentication, the response syntax will be:: + + WWW-Authenticate: Keystone uri="url to Keystone server" + +The client can then make the necessary calls to the Keystone server, obtain a token, and retry the call with the token. + +The token is passed in using ther X-Auth-Token header. + +WSGI API (Headers) +================== + +Upon successful authentication the middleware sends the following +headers to the downstream WSGI app: + +X-Identity-Status + Provides information on whether the request was authenticated or not. + +X-Tenant + Provides the tenant ID (as it appears in the URL in Keystone). This is to support any legacy implementations before Keystone switched to an ID/Name schema for tenants. + +X-Tenant-Id + The unique, immutable tenant Id + +X-Tenant-Name + The unique, but mutable (it can change) tenant name. + +X-User-Id + The user id of the user used to log in + +X-User-Name + The username used to log in + +X-User + The username used to log in. This is to support any legacy implementations before Keystone switched to an ID/Name schema for tenants. + +X-Roles + The roles associated with that user + + +Configuration +============= + +The middleware is configured within the config file of the main application as +a WSGI component. Example for the auth_token middleware:: + + [app:myService] + paste.app_factory = myService:app_factory + + [pipeline:main] + pipeline = + tokenauth + myService + + [filter:tokenauth] + paste.filter_factory = keystone.middleware.auth_token:filter_factory + auth_host = 127.0.0.1 + auth_port = 35357 + auth_protocol = http + auth_uri = http://127.0.0.1:5000/ + admin_token = 999888777666 + ;Uncomment next line and check ip:port to use memcached to cache token requests + ;memcache_hosts = 127.0.0.1:11211 + +*The required configuration entries are:* + +auth_host + The IP address or DNS name of the Keystone server + +auth_port + The TCP/IP port of the Keystone server + +auth_protocol + The protocol of the Keystone server ('http' or 'https') + +auth_uri + The externally accessible URL of the Keystone server. This will be where unauthenticated + clients are redirected to. This is in the form of a URL. For example, if they make an + unauthenticated call, they get this response:: + + HTTP/1.1 401 Unauthorized + Www-Authenticate: Keystone uri='https://auth.example.com/' + Content-Length: 381 + + In this case, the auth_uri setting is set to https://auth.example.com/ + +admin_token + This is the long-lived token issued to the service to authenticate itself when calling + Keystone. See :doc:`configuration` for more information on setting this up. + + +*Optional parameters are:* + +delay_auth_decision + Whether the middleware should reject invalid or unauthenticated calls directly or not. If not, + it will send all calls down to the service to decide, but it will set the HTTP-X-IDENTITY-STATUS + header appropriately (set to'Confirmed' or 'Indeterminate' based on validation) and the + service can then decide if it wants to honor the call or not. This is useful if the service offers + some resources publicly, for example. + +auth_timeout + The amount of time to wait before timing out a call to Keystone (in seconds) + +memcache_hosts + This is used to point to a memcached server (in ip:port format). If supplied, + the middleware will cache tokens and data retrieved from Keystone in memcached + to minimize calls made to Keystone and optimize performance. + +.. warning:: + Tokens are cached for the duration of their validity. If they are revoked eariler in Keystone, + the service will not know and will continue to honor the token as it has them stored in memcached. + Also note that tokens and data stored in memcached are not encrypted. The memcached server must + be trusted and on a secure network. + + +*Parameters needed in a distributed topology.* In this configuration, the middleware is running +on a separate machine or cluster than the protected service (not common - see :doc:`middleware_architecture` +for details on different deployment topologies): + +service_host + The IP address or DNS name of the location of the service (since it is remote + and not automatically down the WSGI chain) + +service_port + The TCP/IP port of the remote service. + +service_protocol + The protocol of the service ('http' or 'https') + +service_pass + The basic auth password used to authenticate to the service (so the service + knows the call is coming from a server that has validated the token and not from + an untrusted source or spoofer) + +service_timeout + The amount of time to wait for the service to respond before timing out. diff --git a/docs/source/middleware_architecture.rst b/docs/source/middleware_architecture.rst new file mode 100644 index 0000000000..a8c38f3cf0 --- /dev/null +++ b/docs/source/middleware_architecture.rst @@ -0,0 +1,529 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +======================= +Middleware Architecture +======================= + +Abstract +======== + +The Keystone middleware architecture supports multiple authentication protocols +in a pluggable manner in OpenStack. By providing support for authentication via +pluggable authentication components, this architecture allows OpenStack +services to be integrated easily into existing deployment environments. It also +provides a path by which to implement support for emerging authentication +standards such as OAUTH. + +Rationale and Goals +=================== + +Keystone is the Identity service for OpenStack. To support the easy integrating +of OpenStack with existing authentication and identity management systems, +Keystone supports talking to multiple backends like LDAP. +And to support different deployment needs, it can support multiple +authentication protocols via pluggable 'authentication components' implemented +as WSGI middleware. + +In this document, we describe the responsibilities of the authentication +middleware. We describe how these interact with underlying OpenStack services +and how existing services can be modified to take advantage of pluggable +authentication. The goal is to allow OpenStack services to be integrated easily +into existing deployment environments and to provide a path by which to +implement support for emerging authentication standards such as OAUTH. + +Specification Overview +====================== + +'Authentication' is the process of determining that users are who they say they +are. Typically, 'authentication protocols' such as HTTP Basic Auth, Digest +Access, public key, token, etc, are used to verify a user's identity. In this +document, we define an ''authentication component'' as a software module that +implements an authentication protocol for an OpenStack service. + +At a high level, an authentication component is simply a reverse proxy that +intercepts HTTP calls from clients. Once it has verified a user's identity, the +authentication component extends the call with information about the current +user and forwards the request to the OpenStack service. Otherwise, if a user's +identity is not verified, the message is rejected before it gets to the +service. This is illustrated in :ref:`authComponent`. + +.. _authComponent: + +Authentication Component +------------------------ + +Figure 1. Authentication Component + +.. image:: images/graphs_authComp.svg + :width: 100% + :height: 180 + :alt: An Authentication Component + +Authentication components may operate in 'delegated mode'. In this mode, the +decision reject an unauthenticated client is delegated to the OpenStack +service. Delegated mode is illustrated in :ref:`authComponentDelegated`. + +Here, requests are forwarded to the OpenStack service with an identity status +message that indicates whether the client's identity has been confirmed or is +indeterminate. It is the OpenStack service that decides whether or not a reject +message should be sent to the client. Note that it is always the responsibility +of the Authentication Component to transmit reject messages to the client. + +.. _authComponentDelegated: + +Authentication Component (Delegated Mode) +----------------------------------------- + +Figure 2. Authentication Component (Delegated Mode) + +.. image:: images/graphs_authCompDelegate.svg + :width: 100% + :height: 180 + :alt: An Authentication Component (Delegated Mode) + +In this architecture, we define interactions between the authentication component +and the OpenStack service. Interactions between the client and the +authentication component are defined only for exceptional cases. For example, +we define the message that should be returned when the OpenStack service is +down. Other interactions, however, are defined by the underlying authentication +protocol and the OpenStack service and are considered out of scope. + +.. _deployStrategies: + +Deployment Strategies +===================== + +An authentication component may be integrated directly into the service +implementation, or it may be deployed separately as an HTTP reverse proxy. This +is illustrated in :ref:`deployment`, showing both approaches to +authentication, labeled Option (a) and Option (b). + +.. _deployment: + +Authentication Component Deployments Options +-------------------------------------------- + +Figure 3. Authentication Component Deployments Options + +.. image:: images/images_layouts.svg + :width: 100% + :height: 180 + :alt: Authentication Component Deployments Options + +In Option (a), the component is integrated into the service implementation. In +this case, communication between the authentication component and the service +can be efficiently implemented via a method call. In Option (b), the component +is deployed separately and communication between the service and the component +involves an HTTP request. In both cases, unauthenticated requests are filtered +before they reach the service. + +Each approach offers some benefits. Option (a) offers low latency and ease of +initial implementation, making it possibly most appropriate as a starting point +for simple configurations. Option (b) offers several key advantages that may be +of particular value in complex and dynamic configurations. It offers the +ability to scale horizontally in cases where authentication is computationally +expensive, such as when verifying digital signatures. Option (b) also allows +authentication components to be written in different programming languages. +Finally, Option (b) allows multiple authentication components to be deployed in +front of the same service. + +OpenStack services can support both embedded (Option (a)) and external (Option +(b)) deployment strategies. Individual authentication components should support +either strategy or they |may| support both strategies. In order to support +option (a), authentication components written in the Python programming +language should be written as WSGI middleware components (in accordance with +the Web Server Gateway Interface (WSGI) standard [PEP-333]_. + +Additionally, services should support the ability to swap between different +embedded or external authentication components via configuration options. + +Exchanging User Information +=========================== + +If a request is successfully authenticated, the authentication component must +extend the request by adding an ``X-Authorization`` header. The header |must| +be formatted as illustrated in :ref:`xAuthHeader`. + +.. _xAuthHeader: + +X-Authorization Header +---------------------- + +Example 1. X-Authorization Header:: + + X-Authorization: Proxy JoeUser + +Here, `Proxy` denotes that the authentication occurred via a proxy (in this +case authentication component) and ''JoeUser'' is the name of the user who +issued the request. + +.. note: + + We considered using an ``Authorization`` header rather than an + ``X-Authorization``, thereby following normal HTTP semantics. There are some + cases, however, where multiple ``Authorization`` headers need to be transmitted + in a single request. We want to assure ourselves that this will not break + common clients before we recommend the approach. + +Authentication components |may| extend the request with additional +information. For example, an authentication system may add additional headers +or modify the target URI to pass authentication information to the back-end +service. Additionally, an authentication component |may| strip sensitive +information — a plain text password, for example — from the request. That said, +an authentication component |should| pass the majority of the request +unmodified. + +Reverse Proxy Authentication +---------------------------- + +An OpenStack service |should| verify that it is receiving requests from a +trusted authentication component. This is particularly important in cases where +the authentication component and the OpenStack service are deployed separately. +In order to trust incoming requests, the OpenStack service should therefore +authenticate the authentication component. To avoid confusion, we call this +'reverse proxy authentication', since in this case the authentication +component is acting as an HTTP reverse proxy. + +Any HTTP-based authentication scheme may be used for reverse proxy +authentication; however, all OpenStack services and all authentication +components |must| support HTTP Basic Authentication as defined in +[RFC-2617]_. + +Whether or not reverse proxy authentication is required is strictly a +deployment concern. For example, an operations team may opt to utilize firewall +rules instead of an authentication protocol to verify the integrity of incoming +request. Because of this, both OpenStack services and authentication components +|must| also allow for unauthenticated communication. + +In cases where reverse proxy authentication is used, the authorization +component may receive an HTTP 401 authentication error or an HTTP 403 +authorization error. These errors indicate that the component does not have +access to the underlying OpenStack service. The authentication component +|must not| return these errors to the client application. Instead, the +component |must| return a 500 internal error. This is illustrated in +:ref:`proxyAuth` and :ref:`proxyAuthDelegated` below. The component +|should| format the errors in a manner that does not break the service +contract defined by the OpenStack service. :ref:`proxyAuthDelegated` +illustrates proxy authorization in delegated mode. Delegated mode is discussed +in detail in the next section. + +.. _proxyAuth: + +Reverse Proxy Authentication +---------------------------- + +Figure 4. Reverse Proxy Authentication + +.. image:: images/graphs_proxyAuth.svg + :width: 100% + :height: 180 + :alt: Reverse Proxy Authentication + +.. _proxyAuthDelegated: + +Reverse Proxy Authentication (Delegated Mode) +--------------------------------------------- + +Figure 5. Reverse Proxy Authentication (Delegated Mode) + +.. image:: images/graphs_delegate_forbiden_proxy.svg + :width: 100% + :height: 180 + :alt: Reverse Proxy Authentication (Delegated Mode) + +Delegated Mode +============== +In some cases, the decision to reject an unauthenticated request should be +delegated to the OpenStack service. An unauthenticated request may be +appropriate in cases when anonymous access is allowed. In order to support +these cases, an authentication component may be placed in Delegated Mode. In +this mode, the component forwards requests to the OpenStack service when the +client's identity has been confirmed or is indeterminate — that is when +credentials are missing. The authentication component directly rejects requests +with invalid credentials. Authentication components |must| extend the +request by adding an `X-Identity-Status` header. The identity status header +|must| contain one of the following values: + +Identity Status Values +---------------------- + +Confirmed + A `confirmed` value indicates that valid credentials were sent and identity + has been confirmed. The service can trust that the request has been sent on + behalf of the user specified in the `X-Authorization` header. + +Indeterminate + An `indeterminate` value indicates that no credentials were sent and + identity has not been confirmed. In this case, the service will receive an + `X-Authorization` header with no user entry as illustrated in + :ref:`xauth-header-indeterminate`. + +.. _xauth-header-indeterminate: + +Indeterminate Identity Headers +------------------------------ + +Example 2. Indeterminate Identity Headers:: + + X-Identity-Status: Indeterminate + X-Authorization: Proxy + +Services |may| reject a delegated request by issuing an HTTP 401 +authentication error or an HTTP 403 authorization error. These responses +|must| contain an ``WWW-Authenticate`` header with a value of ``Delegated`` as +illustrated in :ref:`unauthHeaders`. + +X-Identity-Status + Provides information on whether the request was authenticated or not. + +X-Tenant + Provides the tenant ID (as it appears in the URL in Keystone). This is to support any legacy implementations before Keystone switched to an ID/Name schema for tenants. + +X-Tenant-Id + The unique, immutable tenant Id + +X-Tenant-Name + The unique, but mutable (it can change) tenant name. + +X-User-Id + The user id of the user used to log in + +X-User-Name + The username used to log in + +X-User + The username used to log in. This is to support any legacy implementations before Keystone switched to an ID/Name schema for tenants. + +X-Roles + The roles associated with that user + +.. _unauthHeaders: + +Delegated WWW-Authenticate Header +--------------------------------- + +:: + + WWW-Authenticate: Delegated + +It is important to note that the actual reject message will likely be modified +by the authentication component in order to comply with the authentication +scheme it is implementing. This is illustrated in :ref:`delegateRejectBasic` and +:ref:`delegateRejectOAuth` below. + +.. _delegateRejectBasic: + +Delegated Reject Basic Auth +--------------------------- + +.. image:: images/graphs_delegate_reject_basic.svg + :width: 100% + :height: 180 + :alt: Delegated Reject Basic Auth + +.. _delegateRejectOAuth: + +Delegated Reject OAuth +---------------------- + +.. image:: images/graphs_delegate_reject_oauth.svg + :width: 100% + :height: 180 + :alt: Delegated Reject OAuth + +The presence of the `WWW-Authenticate` header with a value of `Delegated` +distinguishes a client authentication/authorization failure from a component +failure. For example, compare :ref:`delegateForbidden` with :ref:`proxyAuthDelegated`. In +:ref:`delegateForbidden`, the client is not allowed to access the OpenStack service. +In :ref:`proxyAuthDelegated`, it is the authentication component itself which is +unauthorized. + +.. _delegateForbidden: + +Delegated Reject Forbidden +-------------------------- + +Figure 8. Delegated Reject Forbidden + +.. image:: images/graphs_delegate_forbiden_basic.svg + :width: 100% + :height: 180 + :alt: Delegated Reject Forbidden + +Authentication components |must| support both delegated and undelegated +(standard) modes. Delegated mode |should| be configured via a configuration +option. Delegated mode |should| be disabled by default. + +OpenStack services are not required to support delegated mode. If a service +does not support delegated mode, it |must| respond with a 501 not implemented +error and an `WWW-Authenticate` header with a value of `Delegated`. The +authentication component |must not| return the error to the client +application. Instead, the component |must| return a 500 internal error; this is +illustrated in :ref:`delegateUnimplemented`. The component |should| +format the error in a manner that does not break the service contract defined +by the OpenStack service. The component should also log the error such that it +that will inform operators of the misconfiguration. + +.. _delegateUnimplemented: + +Unimplemented Delegated Mode +---------------------------- + +.. image:: images/graphs_delegate_unimplemented.svg + :width: 100% + :height: 180 + :alt: Unimplemented Delegated Mode + +Handling Direct Client Connections +================================== + +Requests from the authentication component to an OpenStack service |must| +contain an ``X-Authorization`` header. If the header is missing, and reverse +proxy authentication fails or is switched off, the OpenStack service |may| +assume that the request is coming directly from a client application. In this +case, the OpenStack service |must| redirect the request to the authentication +component by issuing an HTTP 305 User Proxy redirect. This is illustrated in +:ref:`redirect`. Note that the redirect response |must| include a ``Location`` header +specifying the authentication component's URL as shown in :ref:`redirect-response`. + +.. _redirect: + +Auth Component Redirect +----------------------- + +.. image:: images/graphs_305.svg + :width: 100% + :height: 280 + :alt: Auth Component Redirect + +.. _redirect-response: + +Auth Component Redirect Response +-------------------------------- + +:: + + HTTP/1.1 305 Use Proxy + Date: Thu, 28 Oct 2011 07:41:16 GMT + Location: http://sample.auth.openstack.com/path/to/resource + +Using Multiple Authentication Components +======================================== + +There are some use cases when a service provider might want to consider using +multiple authentication components for different purposes. For instance, a +service provider may have one authentication scheme to authenticate the users +of the service and another one to authenticate the administrators or operations +personnel that maintain the service. For such scenarios, we propose using a +mapper as illustrated in :ref:`multiAuth`. + +.. _multiAuth: + +Multiple Authentication Components +---------------------------------- + +.. image:: images/graphs_mapper.svg + :width: 100% + :height: 320 + :alt: Multiple Authentication Components + +At a high level, a mapper is a simple reverse proxy that intercepts HTTP calls +from clients and routes the request to the appropriate authentication +component. A mapper can make the routing decisions based on a number of routing +rules that map a resource to a specific authentication component. For example, +a request URI may determine whether a call should be authenticated via one +authentication component or another. + +Note that neither the authentication component nor the OpenStack service need +be aware of the mapper. Any external authentication component can be used +alongside others. Mappers may provide a means by which to offer support for +anonymous or guest access to a subset of service resources. A mapper may be +implemented via a traditional reverse proxy server such as Pound or Zeus. + +The Default Component +===================== + +Individual services |must| be distributed with a simple integrated +authentication component by default. Providing such a component lowers barriers +to the deployment of individual services. This is especially important to] +developers who may want to deploy OpenStack services on their own machines. +Also, since there is no direct dependency on an external authentication system, +OpenStack services can be deployed individually, without the need to stand up +and configure additional services. Finally, having a standard authentication +component that all services share promotes a separation of concerns. That is, +as a community we are explicitly stating that services should not develop their +own authentication mechanisms. Additional authentication components may be +developed, of course, but these components should not be intimately coupled to +any one particular service. + +As discussed in :ref:`deployStrategies`, an authentication component may be +integrated directly into the service implementation (Option (a)), or it may be +deployed separately as an HTTP reverse proxy (Option (b)). The default +component should be implemented to support Option (a) and services should +maintain support for Option (b). One way to achieve this is to provide a +method that allows the disabling of the default authentication component via +configuration. This is illustrated in :ref:`both`. Here, requests are +sent directly to the OpenStack service when the default authentication +component is disabled. + +We will discuss the design of the default component in an upcoming blueprint. + +.. _both: + +Disabled Embedded Component +--------------------------- + +.. image:: images/graphs_both.svg + :width: 100% + :height: 250 + :alt: Disabled Embedded Component + +Questions and Answers +===================== + +#. Why do authentication components send reject messages? Why not have + OpenStack services reject requests themselves? + + The content and format of an authentication failed message is determined by + the authentication scheme (or protocol). For the service to respond + appropriately, it would have to be aware of the authentication scheme in + which it participates; this defeats the purpose of pluggable authentication + components. + +#. Why require support for deploying authentication components in separate + nodes? + + The deployment strategy is very flexible. It allows for authentication + components to be horizontally scalable. It allows for components to be written + in different languages. Finally, it allows different authentication components + to be deployed simultaneously as described above. + +References +========== + +.. [PEP-333] pep0333 Phillip J Eby. 'Python Web Server Gateway Interface + v1.0.'' http://www.python.org/dev/peps/pep-0333/. + +.. [RFC-2617] rfc2617 J Franks. P Hallam-Baker. J Hostetler. S Lawrence. + P Leach. A Luotonen. L Stewart. ''HTTP Authentication: Basic and Digest + Access Authentication.'' http://tools.ietf.org/html/rfc2617. + +.. |must| replace:: must must +.. |should| replace:: should should +.. |may| replace:: may may +.. |must not| replace:: "must not" "must not" + diff --git a/docs/source/migration.rst b/docs/source/migration.rst new file mode 100644 index 0000000000..460d980b9f --- /dev/null +++ b/docs/source/migration.rst @@ -0,0 +1,126 @@ +=================== +Database Migrations +=================== + +Keystone uses SQLAlchemy Migrate (``sqlalchemy-migrate``) to manage +migrations. + +Migrations are tracked using a metadata table (``migrate_version``), which +allows keystone to compare the state of your database to the state it +expects, and to move between versions. + +.. WARNING:: + + Backup your database before applying migrations. Migrations may + attempt to modify both your schema and data, and could result in data + loss. + + Always review the behavior of migrations in a staging environment + before applying them in production. + +Getting Started +=============== + +Your initial approach to migrations should depend on whether you have an +empty database or a schema full of data. + +Starting with an empty database +------------------------------- + +If you have an empty database for keystone to work with, you can simply +run:: + + $ ./bin/keystone-manage database sync + +This command will initialize your metadata table, and run through all the +schema & data migrations necessary to bring your database in sync with +keystone. That's it! + +Starting with an existing database +---------------------------------- + +Place an existing database under version control to enable migration +support:: + + $ ./bin/keystone-manage database version_control + +This command simply creates a ``migrate_version`` table, set at +``version_number`` 0, which indicates that no migrations have been applied. + +If you are starting with an existing schema, you can jump to a specific +schema version without performing migrations using the ``database goto`` +command. For example, if you're starting from a diablo-compatible +database, set your current database ``version_number`` to ``1`` using:: + + $ ./bin/keystone-manage database goto + +Determine your appropriate database ``version_number`` by referencing the +following table: + + +------------+-------------+ + | Release | ``version`` | + +============+=============+ + | pre-diablo | (see below) | + +------------+-------------+ + | diablo | 1 | + +------------+-------------+ + | essex-m1 | 3 | + +------------+-------------+ + | essex-m2 | 4 | + +------------+-------------+ + +From there, you can upgrade normally (see :ref:`upgrading`). + +Starting with a pre-diablo database (cactus) +-------------------------------------------- + +You'll need to manually migrate your database to a diablo-compatible +schema, and continue forward from there (if desired) using migrations. + +.. _upgrading: + +Upgrading & Downgrading +======================= + +.. note:: + + Attempting to start keystone with an outdated schema will cause + keystone to abort, to avoid corrupting your data. + +Upgrade to the latest version automatically:: + + $ ./bin/keystone-manage database sync + +Check your current schema version:: + + $ ./bin/keystone-manage database version + +Jump to a specific version without performing migrations:: + + $ ./bin/keystone-manage database goto + +Upgrade to a specific version:: + + $ ./bin/keystone-manage database upgrade + +Downgrade to a specific version (will likely result in data loss!):: + + $ ./bin/keystone-manage database downgrade + +Opting Out of Migrations +======================== + +If you don't want to use migrations (e.g. if you want to manage your +schema manually), keystone will complain in your logs on startup, but +won't actually stop you from doing so. + +It's recommended that you use migrations to get up and running, but if +you want to manage migrations manually after that, simply drop the +``migrate_version`` table:: + + DROP TABLE migrate_version; + +Useful Links +============ + +Principles to follow when developing migrations `OpenStack Deployability `_ diff --git a/docs/source/nova-api-paste.rst b/docs/source/nova-api-paste.rst new file mode 100644 index 0000000000..586bac72ed --- /dev/null +++ b/docs/source/nova-api-paste.rst @@ -0,0 +1,142 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +nova-api-paste example +====================== +:: + + ####### + # EC2 # + ####### + + [composite:ec2] + use = egg:Paste#urlmap + /: ec2versions + /services/Cloud: ec2cloud + /services/Admin: ec2admin + /latest: ec2metadata + /2007-01-19: ec2metadata + /2007-03-01: ec2metadata + /2007-08-29: ec2metadata + /2007-10-10: ec2metadata + /2007-12-15: ec2metadata + /2008-02-01: ec2metadata + /2008-09-01: ec2metadata + /2009-04-04: ec2metadata + /1.0: ec2metadata + + [pipeline:ec2cloud] + pipeline = logrequest totoken authtoken keystonecontext cloudrequest authorizer ec2executor + + [pipeline:ec2admin] + pipeline = logrequest totoken authtoken keystonecontext adminrequest authorizer ec2executor + + [pipeline:ec2metadata] + pipeline = logrequest ec2md + + [pipeline:ec2versions] + pipeline = logrequest ec2ver + + [filter:logrequest] + paste.filter_factory = nova.api.ec2:RequestLogging.factory + + [filter:ec2lockout] + paste.filter_factory = nova.api.ec2:Lockout.factory + + [filter:totoken] + paste.filter_factory = keystone.middleware.ec2_token:EC2Token.factory + + [filter:ec2noauth] + paste.filter_factory = nova.api.ec2:NoAuth.factory + + [filter:authenticate] + paste.filter_factory = nova.api.ec2:Authenticate.factory + + [filter:cloudrequest] + controller = nova.api.ec2.cloud.CloudController + paste.filter_factory = nova.api.ec2:Requestify.factory + + [filter:adminrequest] + controller = nova.api.ec2.admin.AdminController + paste.filter_factory = nova.api.ec2:Requestify.factory + + [filter:authorizer] + paste.filter_factory = nova.api.ec2:Authorizer.factory + + [app:ec2executor] + paste.app_factory = nova.api.ec2:Executor.factory + + [app:ec2ver] + paste.app_factory = nova.api.ec2:Versions.factory + + [app:ec2md] + paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory + + ############# + # Openstack # + ############# + + [composite:osapi] + use = egg:Paste#urlmap + /: osversions + /v1.1: openstackapi + + [pipeline:openstackapi] + pipeline = faultwrap authtoken keystonecontext ratelimit extensions osapiapp + + [filter:faultwrap] + paste.filter_factory = nova.api.openstack:FaultWrapper.factory + + [filter:auth] + paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory + + [filter:noauth] + paste.filter_factory = nova.api.openstack.auth:NoAuthMiddleware.factory + + [filter:ratelimit] + paste.filter_factory = nova.api.openstack.limits:RateLimitingMiddleware.factory + + [filter:extensions] + paste.filter_factory = nova.api.openstack.extensions:ExtensionMiddleware.factory + + [app:osapiapp] + paste.app_factory = nova.api.openstack:APIRouter.factory + + [pipeline:osversions] + pipeline = faultwrap osversionapp + + [app:osversionapp] + paste.app_factory = nova.api.openstack.versions:Versions.factory + + ########## + # Shared # + ########## + + [filter:keystonecontext] + paste.filter_factory = keystone.middleware.nova_keystone_context:NovaKeystoneContext.factory + + [filter:authtoken] + paste.filter_factory = keystone.middleware.auth_token:filter_factory + service_protocol = http + service_host = 127.0.0.1 + service_port = 5000 + auth_host = 127.0.0.1 + auth_port = 35357 + auth_protocol = http + auth_uri = http://127.0.0.1:5000/ + admin_token = 999888777666 + ;Uncomment next line and check ip:port to use memcached to cache token requests + ;memcache_hosts = 127.0.0.1:11211 diff --git a/docs/source/releases.rst b/docs/source/releases.rst new file mode 100644 index 0000000000..a4b698d752 --- /dev/null +++ b/docs/source/releases.rst @@ -0,0 +1,36 @@ +============= +Release notes +============= + + +E3 (January 26, 2012) +========================================== +* Contract compliance: version response and ATOM, 300 multiple choice +* Global endpoints returned for unscoped calls +* adminUrl only shown to admin clients +* Endpoints have unique ID +* Auth-N/Auth-Z for S3 API (OS-KSS3 extension) +* Default tenant scope optionally returned when authenticating +* Vary header returned for caching proxies + +* Portable identifiers: modifiable, string identifiers in database backend +* Much improved keystone-manage command (see --help and docs) +* OS-KSVALIDATE extension to support not passing tokens in URL +* OS-KSEC2 and OS-KSS3 extensions respond on /tokens +* HP-IDM extension to filter roles to a given service ID +* Additional caching options in middleware (memcache and swift cache) + +* Enhanced configuration management (in line with other OpenStack projects) +* Additional logging +* Enhanced tracer tool (-t or --trace-calls) + +See comprehensive list here https://launchpad.net/keystone/+milestone/essex-3 + + +E2 (December 15, 2011) +======================== +* D5 compatibility middleware +* Database versioning +* Much more documentation: http://keystone.openstack.org + +See https://launchpad.net/keystone/+milestone/essex-2 diff --git a/docs/source/serviceAPI_curl_examples.rst b/docs/source/serviceAPI_curl_examples.rst new file mode 100644 index 0000000000..d05afc9ff3 --- /dev/null +++ b/docs/source/serviceAPI_curl_examples.rst @@ -0,0 +1,69 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +=============================== +Service API Examples Using Curl +=============================== + +The service API is defined to be a subset of the Admin API and, by +default, runs on port 5000. + +GET / +===== + +This call is identical to that documented for the Admin API, except +that it uses port 5000, instead of port 35357, by default:: + + $ curl http://0.0.0.0:5000 + +or:: + + $ curl http://0.0.0.0:5000/v2.0/ + +See the `Admin API Examples Using Curl`_ for more info. + +.. _`Admin API Examples Using Curl`: adminAPI_curl_examples.html + +GET /extensions +=============== + +This call is identical to that documented for the Admin API. + +POST /tokens +============ + +This call is identical to that documented for the Admin API. + +GET /tenants +============ + +List all of the tenants your token can access:: + + $ curl -H "X-Auth-Token:887665443383838" http://localhost:5000/v2.0/tenants + +Returns:: + + { + "tenants_links": [], + "tenants": [ + { + "enabled": true, + "description": "None", + "name": "customer-x", + "id": "1" + } + ] + } diff --git a/docs/source/services.rst b/docs/source/services.rst new file mode 100644 index 0000000000..d1c33381d6 --- /dev/null +++ b/docs/source/services.rst @@ -0,0 +1,92 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +================ +Services +================ + +.. toctree:: + :maxdepth: 1 + + +What are services? +================== + +Keystone includes service registry and service catalog functionality which it +uses to respond to client authentication requests with information useful to +clients in locating the list of available services they can access. + +The Service entity in Keystone represents an OpenStack service that is integrated +with Keystone. The Service entity is also used as a reference from roles, endpoints, +and endpoint templates. + +Keystone also includes an authorization mechanism to allow a service to own +its own roles and endpoints and prevent other services from changing or +modifying them. + +Who can create services? +======================== + +Any user with the Admin or Service Admin roles in Keystone may create services. + +How are services created? +========================= + +Services can be created using ``keystone-manage`` or through the REST API using +the OS-KSADM extension calls. + +Using ``keystone-manage`` (see :doc:`man/keystone-manage` for details):: + + $ keystone-manage add service compute nova 'This is a sample compute service' + +Using the REST API (see `extensions dev guide `_ for details):: + + $ curl -H "Content-type: application/json" -X POST -d '{ + "OS-KSADM:service": { + "name": "nova", + "type": "compute", + "description": "This is a sample compute service" + } + }' -H "X-Auth-Token: 999888777666" http://localhost:35357/v2.0/OS-KSADM/services/ + +How is service ownership determined? +==================================== + +Currently, the way to assign ownership to a service is to provide the owner's +user id in the keystone-manage add command:: + + $ keystone-manage add service nova compute 'This is a sample compute service' joeuser + +This will assign ownership to the new service to joeuser. + +When a service has an owner, then only that owner (or a global Admin) can create and manage +roles that start with that service name (ex: "nova:admin") and manage endpoints +and endpoint templates associated with that service. + +Listing services +================ + +Using ``keystone-manage``, the list of services and their owners can be listed:: + + $ keystone-manage service list + + id name type owner_id description + ------------------------------------------------------------------------------- + 1 compute nova joeuser This is a sample compute service + +Using the REST API, call ``GET /v2.0/OS-KSADM/services`` + +.. note: The rest API does not yet support service ownership diff --git a/docs/source/ssl.rst b/docs/source/ssl.rst new file mode 100644 index 0000000000..839e951ea7 --- /dev/null +++ b/docs/source/ssl.rst @@ -0,0 +1,118 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +=========================== +x.509 Client Authentication +=========================== + +Purpose +======= + +Allows the Keystone middleware to authenticate itself with the Keystone server +via an x.509 client certificate. Both Service API and Admin API may be secured +with this feature. + +Certificates +============ + +The following types of certificates are required. A set of certficates is provided +in the examples/ssl directory with the Keystone distribution for testing. Here +is the description of each of them and their purpose: + +ca.pem + Certificate Authority chain to validate against. + +keystone.pem + Public certificate for Keystone server. + +middleware-key.pem + Public and private certificate for Keystone middleware. + +cakey.pem + Private key for the CA. + +keystonekey.pem + Private key for the Keystone server. + +Note that you may choose whatever names you want for these certificates, or combine +the public/private keys in the same file if you wish. These certificates are just +provided as an example. + +Configuration +============= + +By default, the Keystone server does not use SSL. To enable SSL with client authentication, +modify the etc/keystone.conf file accordingly: + +1. To enable SSL for Service API:: + + service_ssl = True + +2. To enable SSL for Admin API:: + + admin_ssl = True + +3. To enable SSL client authentication:: + + cert_required = True + +4. Set the location of the Keystone certificate file (example):: + + certfile = /etc/keystone/ca/certs/keystone.pem + +5. Set the location of the Keystone private file (example):: + + keyfile = /etc/keystone/ca/private/keystonekey.pem + +6. Set the location of the CA chain:: + + ca_certs = /etc/keystone/ca/certs/ca.pem + +Middleware +========== + +Add the following to your middleware configuration to support x.509 client authentication. +If ``cert_required`` is set to ``False`` on the keystone server, the certfile and keyfile parameters +in steps 3) and 4) may be commented out. + +1. Specify 'https' as the auth_protocol:: + + auth_protocol = https + +2. Modify the protocol in 'auth_uri' to be 'https' as well, if the service API is configured + for SSL:: + + auth_uri = https://localhost:5000/ + +3. Set the location of the middleware certificate file (example):: + + certfile = /etc/keystone/ca/certs/middleware-key.pem + +4. Set the location of the Keystone private file (example):: + + keyfile = /etc/keystone/ca/certs/middleware-key.pem + +For an example, take a look at the ``echo.ini`` middleware configuration for the 'echo' example +service in the examples/echo directory. + +Testing +======= + +You can test out how it works by using the ``echo`` example service in the ``examples/echo`` directory +and the certficates included in the ``examples/ssl`` directory. Invoke the ``echo_client.py`` with +the path to the client certificate:: + + python echo_client.py -s diff --git a/docs/source/usingkeystone.rst b/docs/source/usingkeystone.rst new file mode 100644 index 0000000000..bb52a94d09 --- /dev/null +++ b/docs/source/usingkeystone.rst @@ -0,0 +1,28 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============== +Using Keystone +============== + +Curl examples +------------- + +.. toctree:: + :maxdepth: 1 + + adminAPI_curl_examples + serviceAPI_curl_examples From e643f239816bae29e8206407db3d5eabdd2ea4b0 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 10:57:02 -0800 Subject: [PATCH 261/334] doc updates --- docs/source/api_curl_examples.rst | 442 ++++++++++++++++++++++++++++ docs/source/architecture.rst | 4 + docs/source/community.rst | 26 +- docs/source/configuration.rst | 222 +++++++++++--- docs/source/developing.rst | 130 ++++---- docs/source/index.rst | 20 +- docs/source/man/keystone-manage.rst | 265 ++++++++++------- docs/source/man/keystone.rst | 69 ++--- docs/source/setup.rst | 73 +++-- docs/source/testing.rst | 77 ----- keystone/cli.py | 2 +- keystone/middleware/core.py | 38 +++ 12 files changed, 969 insertions(+), 399 deletions(-) create mode 100644 docs/source/api_curl_examples.rst delete mode 100644 docs/source/testing.rst diff --git a/docs/source/api_curl_examples.rst b/docs/source/api_curl_examples.rst new file mode 100644 index 0000000000..686e8bd59c --- /dev/null +++ b/docs/source/api_curl_examples.rst @@ -0,0 +1,442 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +=============================== +Service API Examples Using Curl +=============================== + +The service API is defined to be a subset of the Admin API and, by +default, runs on port 5000. + +GET / +===== + +This call is identical to that documented for the Admin API, except +that it uses port 5000, instead of port 35357, by default:: + + $ curl http://0.0.0.0:5000 + +or:: + + $ curl http://0.0.0.0:5000/v2.0/ + +See the `Admin API Examples Using Curl`_ for more info. + +.. _`Admin API Examples Using Curl`: adminAPI_curl_examples.html + +GET /extensions +=============== + +This call is identical to that documented for the Admin API. + +POST /tokens +============ + +This call is identical to that documented for the Admin API. + +GET /tenants +============ + +List all of the tenants your token can access:: + + $ curl -H "X-Auth-Token:887665443383838" http://localhost:5000/v2.0/tenants + +Returns:: + + { + "tenants_links": [], + "tenants": [ + { + "enabled": true, + "description": "None", + "name": "customer-x", + "id": "1" + } + ] + } + +============================= +Admin API Examples Using Curl +============================= + +These examples assume a default port value of 35357, and depend on the +``sampledata`` bundled with keystone. + +GET / +===== + +Disover API version information, links to documentation (PDF, HTML, WADL), +and supported media types:: + + $ curl http://0.0.0.0:35357 + +or:: + + $ curl http://0.0.0.0:35357/v2.0/ + +Returns:: + + { + "version":{ + "id":"v2.0", + "status":"beta", + "updated":"2011-11-19T00:00:00Z", + "links":[ + { + "rel":"self", + "href":"http://127.0.0.1:35357/v2.0/" + }, + { + "rel":"describedby", + "type":"text/html", + "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/content/" + }, + { + "rel":"describedby", + "type":"application/pdf", + "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/identity-dev-guide-2.0.pdf" + }, + { + "rel":"describedby", + "type":"application/vnd.sun.wadl+xml", + "href":"http://127.0.0.1:35357/v2.0/identity-admin.wadl" + } + ], + "media-types":[ + { + "base":"application/xml", + "type":"application/vnd.openstack.identity-v2.0+xml" + }, + { + "base":"application/json", + "type":"application/vnd.openstack.identity-v2.0+json" + } + ] + } + } + +GET /extensions +=============== + +Discover the API extensions enabled at the endpoint:: + + $ curl http://0.0.0.0:35357/extensions + +Returns:: + + { + "extensions":{ + "values":[] + } + } + +POST /tokens +============ + +Authenticate by exchanging credentials for an access token:: + + $ curl -d '{"auth":{"passwordCredentials":{"username": "joeuser", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:35357/v2.0/tokens + +Returns:: + + { + "access":{ + "token":{ + "expires":"2012-02-05T00:00:00", + "id":"887665443383838", + "tenant":{ + "id":"1", + "name":"customer-x" + } + }, + "serviceCatalog":[ + { + "endpoints":[ + { + "adminURL":"http://swift.admin-nets.local:8080/", + "region":"RegionOne", + "internalURL":"http://127.0.0.1:8080/v1/AUTH_1", + "publicURL":"http://swift.publicinternets.com/v1/AUTH_1" + } + ], + "type":"object-store", + "name":"swift" + }, + { + "endpoints":[ + { + "adminURL":"http://cdn.admin-nets.local/v1.1/1", + "region":"RegionOne", + "internalURL":"http://127.0.0.1:7777/v1.1/1", + "publicURL":"http://cdn.publicinternets.com/v1.1/1" + } + ], + "type":"object-store", + "name":"cdn" + } + ], + "user":{ + "id":"1", + "roles":[ + { + "tenantId":"1", + "id":"3", + "name":"Member" + } + ], + "name":"joeuser" + } + } + } + +.. note:: + + Take note of the value ['access']['token']['id'] value produced here (``887665443383838``, above), as you can use it in the calls below. + +GET /tokens/{token_id} +====================== + +.. note:: + + This call refers to a token known to be valid, ``887665443383838`` in this case. + +Validate a token:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 + +If the token is valid, returns:: + + { + "access":{ + "token":{ + "expires":"2012-02-05T00:00:00", + "id":"887665443383838", + "tenant":{ + "id":"1", + "name":"customer-x" + } + }, + "user":{ + "name":"joeuser", + "tenantName":"customer-x", + "id":"1", + "roles":[ + { + "serviceId":"1", + "id":"3", + "name":"Member" + } + ], + "tenantId":"1" + } + } + } + +HEAD /tokens/{token_id} +======================= + +This is a high-performance variant of the GET call documented above, which +by definition, returns no response body:: + + $ curl -I -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 + +... which returns ``200``, indicating the token is valid:: + + HTTP/1.1 200 OK + Content-Length: 0 + Content-Type: None + Date: Tue, 08 Nov 2011 23:07:44 GMT + +GET /tokens/{token_id}/endpoints +================================ + +List all endpoints for a token:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838/endpoints + +Returns:: + + { + "endpoints_links": [ + { + "href": "http://127.0.0.1:35357/tokens/887665443383838/endpoints?'marker=5&limit=10'", + "rel": "next" + } + ], + "endpoints": [ + { + "internalURL": "http://127.0.0.1:8080/v1/AUTH_1", + "name": "swift", + "adminURL": "http://swift.admin-nets.local:8080/", + "region": "RegionOne", + "tenantId": 1, + "type": "object-store", + "id": 1, + "publicURL": "http://swift.publicinternets.com/v1/AUTH_1" + }, + { + "internalURL": "http://localhost:8774/v1.0", + "name": "nova_compat", + "adminURL": "http://127.0.0.1:8774/v1.0", + "region": "RegionOne", + "tenantId": 1, + "type": "compute", + "id": 2, + "publicURL": "http://nova.publicinternets.com/v1.0/" + }, + { + "internalURL": "http://localhost:8774/v1.1", + "name": "nova", + "adminURL": "http://127.0.0.1:8774/v1.1", + "region": "RegionOne", + "tenantId": 1, + "type": "compute", + "id": 3, + "publicURL": "http://nova.publicinternets.com/v1.1/ + }, + { + "internalURL": "http://127.0.0.1:9292/v1.1/", + "name": "glance", + "adminURL": "http://nova.admin-nets.local/v1.1/", + "region": "RegionOne", + "tenantId": 1, + "type": "image", + "id": 4, + "publicURL": "http://glance.publicinternets.com/v1.1/" + }, + { + "internalURL": "http://127.0.0.1:7777/v1.1/1", + "name": "cdn", + "adminURL": "http://cdn.admin-nets.local/v1.1/1", + "region": "RegionOne", + "tenantId": 1, + "versionId": "1.1", + "versionList": "http://127.0.0.1:7777/", + "versionInfo": "http://127.0.0.1:7777/v1.1", + "type": "object-store", + "id": 5, + "publicURL": "http://cdn.publicinternets.com/v1.1/1" + } + ] + } + +GET /tenants +============ + +List all of the tenants in the system (requires an Admin ``X-Auth-Token``):: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants + +Returns:: + + { + "tenants_links": [], + "tenants": [ + { + "enabled": false, + "description": "None", + "name": "project-y", + "id": "3" + }, + { + "enabled": true, + "description": "None", + "name": "ANOTHER:TENANT", + "id": "2" + }, + { + "enabled": true, + "description": "None", + "name": "customer-x", + "id": "1" + } + ] + } + +GET /tenants/{tenant_id} +======================== + +Retrieve information about a tenant, by tenant ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1 + +Returns:: + + { + "tenant":{ + "enabled":true, + "description":"None", + "name":"customer-x", + "id":"1" + } + } + +GET /tenants/{tenant_id}/users/{user_id}/roles +============================================== + +List the roles a user has been granted on a tenant:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1/users/1/roles + +Returns:: + + { + "roles_links":[], + "roles":[ + { + "id":"3", + "name":"Member" + } + ] + } + +GET /users/{user_id} +==================== + +Retrieve information about a user, by user ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/1 + +Returns:: + + { + "user":{ + "tenantId":"1", + "enabled":true, + "id":"1", + "name":"joeuser" + } + } + +GET /users/{user_id}/roles +========================== + +Retrieve the roles granted to a user, given a user ID:: + + $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/4/roles + +Returns:: + + { + "roles_links":[], + "roles":[ + { + "id":"2", + "name":"KeystoneServiceAdmin" + } + ] + } diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index 8de4550256..21d9b895a5 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -17,6 +17,10 @@ Keystone Architecture ===================== +Much of the design is precipitated from the expectation that the auth backends +for most deployments will actually be shims in front of existing user systems. + +.........JOEEDIT.......... Keystone has two major components: Authentication and a Service Catalog. Authentication diff --git a/docs/source/community.rst b/docs/source/community.rst index bbad242147..d3e3217870 100644 --- a/docs/source/community.rst +++ b/docs/source/community.rst @@ -33,20 +33,6 @@ from blueprint designs to documentation to testing to deployment scripts. .. _Launchpad: https://launchpad.net/keystone .. _wiki: http://wiki.openstack.org/ - - -Contributing Code ------------------ - -To contribute code, sign up for a Launchpad account and sign a contributor license agreement, -available on the ``_. Once the CLA is signed you -can contribute code through the Gerrit version control system which is related to your Launchpad account. - -To contribute tests, docs, code, etc, refer to our `Gerrit-Jenkins-Github Workflow`_. - -.. _`Gerrit-Jenkins-Github Workflow`: http://wiki.openstack.org/GerritJenkinsGithub - - #openstack on Freenode IRC Network ---------------------------------- @@ -68,10 +54,10 @@ to write drafts for specs or documentation, describe a blueprint, or collaborate Keystone on Launchpad --------------------- -Launchpad is a code hosting service that hosts the Keystone source code. From -Launchpad you can report bugs, ask questions, and register blueprints (feature requests). +Launchpad is a code hosting that OpenStack is using to track bugs, feature work, and releases of OpenStack. Like other OpenStack projects, Keystone source code is hosted on GitHub -* `Launchpad Keystone Page `_ +* `Keystone Project Page on Launchpad `_ +* `Keystone Source Repository on GitHub `_ OpenStack Blog -------------- @@ -82,9 +68,9 @@ events and posts from OpenStack contributors. `OpenStack Blog `_ -See also: `Planet OpenStack `_, aggregating blogs -about OpenStack from around the internet into a single feed. If you'd like to contribute to this blog -aggregation with your blog posts, there are instructions for `adding your blog `_. +See also: `Planet OpenStack `_, an aggregation of blogs +about OpenStack from around the internet, combined into a web site and RSS feed. If you'd like to +contribute with your blog posts, there are instructions for `adding your blog `_. Twitter ------- diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index a98d92f88c..02fbd4b44f 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -21,80 +21,214 @@ Configuring Keystone .. toctree:: :maxdepth: 1 - keystone.conf man/keystone-manage -Once Keystone is installed, there are a number of configuration options -available and potentially some initial data to create and set up. - -Sample data / Quick Setup -========================= - -Default sampledata is provided for easy setup and testing in bin/sampeldata. To -set up the sample data run the following command while Keystone is running:: - - $ ./bin/sampledata - -The sample data created comes from the file :doc:`sourcecode/keystone.test.sampledata` +Once Keystone is installed, it is configured via a primary configuration file +(:doc:`keystone.conf`), possibly a separate logging configuration file, and +initializing data into keystone using the command line client. Keystone Configuration File =========================== -Most configuration is done via configuration files. The default files are -in ``/etc/keystone.conf`` +The keystone configuration file is an 'ini' file format with sections, +extended from Paste_, a common system used to configure python WSGI based +applications. In addition to the paste config entries, general configuration +values are stored under [DEFAULT] and [sql], and then drivers for the various +backend components are included under their individual sections. -When starting up a Keystone server, you can specify the configuration file to -use (see :doc:`controllingservers`). -If you do **not** specify a configuration file, keystone will look in the following -directories for a configuration file, in order: +The driver sections include: +* ``[identity]`` - the python module that backends the identity system +* ``[catalog]`` - the python module that backends the service catalog +* ``[token]`` - the python module that backends the token providing mechanisms +* ``[policy]`` - the python module that drives the policy system for RBAC +* ``[ec2]`` - the python module providing the EC2 translations for OpenStack + +The keystone configuration file is expected to be named ``keystone.conf``. +When starting up Keystone, you can specify a different configuration file to +use with ``--config-file``. If you do **not** specify a configuration file, +keystone will look in the following directories for a configuration file, in +order: * ``~/.keystone`` * ``~/`` * ``/etc/keystone`` * ``/etc`` -The keystone configuration file should be named ``keystone.conf``. -If you installed keystone via your operating system's -package management system, it is likely that you will have sample -configuration files installed in ``/etc/keystone``. +Logging is configured externally to the rest of keystone, the file specifying +the logging configuration is in the [DEFAULT] section of the keystone conf +file under ``log_config``. If you wish to route all your logging through +syslog, there is a ``use_syslog`` option also in the [DEFAULT] section that +easy. -In addition to this documentation page, you can check the -``etc/keystone.conf`` sample configuration -files distributed with keystone for example configuration files for each server -application with detailed comments on what each options does. +A sample logging file is available with the project in the directory ``etc/logging.conf.sample``. Like other OpenStack projects, keystone uses the +`python logging module`, which includes extensive configuration options for +choosing the output levels and formats. + +In addition to this documentation page, you can check the ``etc/keystone.conf`` +sample configuration files distributed with keystone for example configuration +files for each server application. + +.. _Paste: http://pythonpaste.org/ +.. _`python logging module`: http://docs.python.org/library/logging.html Sample Configuration Files -------------------------- -Keystone ships with sample configuration files in keystone/etc. These files are: +* ``etc/keystone.conf`` +* ``etc/logging.conf.sample`` -1. keystone.conf +Initializing Keystone +===================== - A standard configuration file for running keystone in stand-alone mode. - It has a set of default extensions loaded to support administering Keystone - over REST. It uses a local SQLite database. +Keystone must be running in order to initialize data within it. This is because +the keystone-manage commands are all used the same REST API that other +OpenStack systems utilize. -2. memcache.conf +General keystone-manage options: +-------------------------------- - A configuration that uses memcached for storing tokens (but still SQLite for all - other entities). This requires memcached running. +* ``--id-only`` : causes ``keystone-manage`` to return only the UUID result +from the API call. +* ``--endpoint`` : allows you to specify the keystone endpoint to communicate with. The default endpoint is http://localhost:35357/v2.0' +* ``--auth-token`` : provides the authorization token -3. ssl.conf +``keystone-manage`` is set up to expect commands in the general form of ``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to provide additional information to the command. For example, the command +``tenant`` has the subcommand ``create``, which takes the required keyword ``tenant_name``:: - A configuration that runs Keystone with SSL (so all URLs are accessed over HTTPS). + keystone-manage tenant create tenant_name=example_tenant -To run any of these configurations, use the `-c` option:: +Invoking keystone-manage by itself will give you some usage information. - ./keystone -c ../etc/ssl.conf +Available keystone-manage commands: + db_sync: Sync the database. + ec2: no docs + role: Role CRUD functions. + service: Service CRUD functions. + tenant: Tenant CRUD functions. + token: Token CRUD functions. + user: User CRUD functions. +Tenants +------- +Tenants are the high level grouping within Keystone that represent groups of +users. A tenant is the grouping that owns virtual machines within Nova, or +containers within Swift. A tenant can have zero or more users, Users can be assocaited with more than one tenant, and each tenant - user pairing can have a role associated with it. -Usefule Links -------------- +* tenant create -For a sample configuration file with explanations of the settings, see :doc:`keystone.conf` + keyword arguments + * tenant_name + * id (optional) -For configuring an LDAP backend, see http://mirantis.blogspot.com/2011/08/ldap-identity-store-for-openstack.html +example:: + keystone-manage --id-only tenant create tenant_name=admin -For configuration settings of middleware components, see :doc:`middleware` \ No newline at end of file +creates a tenant named "admin". + +* tenant delete + + keyword arguments + * tenant_id + +example:: + keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 + +* tenant update + + keyword arguments + * description + * name + * tenant_id + +example:: + keystone-manage tenant update \ + tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 \ + description="those other guys" \ + name=tog + +Users +----- + +* user create + + keyword arguments + * name + * password + * email + +example:: + keystone-manage user --ks-id-only create \ + name=admin \ + password=secrete \ + email=admin@example.com + +* user delete + + keyword arguments + +* user list + + keyword arguments + +* user update_email + + keyword arguments + +* user update_enabled + + keyword arguments + +* user update_password + + keyword arguments + +* user update_tenant + + keyword arguments + +Roles +----- + +* role create + + keyword arguments + * name + +exmaple:: + keystone-manage role --ks-id-only create name=Admin + +* role add_user_to_tenant + + keyword arguments + * role_id + * user_id + * tenant_id + +example:: + + keystone-manage role add_user_to_tenant \ + role_id=19d1d3344873464d819c45f521ff9890 \ + user_id=08741d8ed88242ca88d1f61484a0fe3b \ + tenant_id=20601a7f1d94447daa4dff438cb1c209 + +* role remove_user_from_tenant + +* role get_user_role_refs + +Services +-------- + +* service create + + keyword arguments + * name + * service_type + * description + +example:: + keystone-manage service create \ + name=nova \ + service_type=compute \ + description="Nova Compute Service" diff --git a/docs/source/developing.rst b/docs/source/developing.rst index acd2273068..b19fec2455 100644 --- a/docs/source/developing.rst +++ b/docs/source/developing.rst @@ -18,33 +18,22 @@ Developing with Keystone ======================== -Get your development environment set up according to :doc:`setup`. +Contributing Code +================= -Running a development instance -============================== +To contribute code, sign up for a Launchpad account and sign a contributor license agreement, +available on the ``_. Once the CLA is signed you +can contribute code through the Gerrit version control system which is related to your Launchpad account. -Setting up a virtualenv ------------------------ +To contribute tests, docs, code, etc, refer to our `Gerrit-Jenkins-Github Workflow`_. -We recommend establishing a virtualenv to run keystone within. To establish -this environment, use the command:: +.. _`Gerrit-Jenkins-Github Workflow`: http://wiki.openstack.org/GerritJenkinsGithub - $ python tools/install_venv.py +Setup +----- -This will create a local virtual environment in the directory ``.venv``. -Once created, you can activate this virtualenv for your current shell using:: - - $ source .venv/bin/activate - -The virtual environment can be disabled using the command:: - - $ deactivate - -You can also use ``tools\with_venv.sh`` to prefix commands so that they run -within the virtual environment. For more information on virtual environments, -see virtualenv_. - -.. _virtualenv: http://www.virtualenv.org/ +Get your development environment set up according to :doc:`setup`. The instructions from here will +assume that you have installed keystone into a virtualenv. If you chose not to, simply exclude "tools/with_venv.sh" from the example commands below. Running Keystone ---------------- @@ -52,84 +41,69 @@ Running Keystone To run the keystone Admin and API server instances, use:: $ tools/with_venv.sh bin/keystone - -Running a demo service that uses Keystone ------------------------------------------ - -To run client demo (with all auth middleware running locally on sample service):: - - $ tools/with_venv.sh examples/echo/bin/echod - -which spins up a simple "echo" service on port 8090. To use a simple echo client:: - - $ python examples/echo/echo_client.py + +this runs keystone with the configuration the etc/ directory of the project. See :doc:`configuration` for details on how Keystone is configured. Interacting with Keystone -========================= +------------------------- You can interact with Keystone through the command line using :doc:`man/keystone-manage` which allows you to establish tenants, users, etc. You can also interact with Keystone through it's REST API. There is a python -keystone client library python-keystoneclient_ which interacts exclusively through -the REST API. +keystone client library `python-keystoneclient`_ which interacts exclusively through +the REST API, and which keystone itself uses to provide it's command-line interface. -.. _python-keystoneclient: https://github.com/4P/python-keystoneclient +.. _`python-keystoneclient`: https://github.com/openstack/python-keystoneclient -The easiest way to establish some base information in Keystone to interact with is -to invoke:: +Running Tests +============= - $ tools/with_venv.sh bin/sampledata +To run the full suites of tests maintained within Keystone, run:: -You can see the details of what that creates in ``keystone/test/sampledata.py`` + $ ./run_tests.sh -Enabling debugging middleware ------------------------------ +This shows realtime feedback during test execution, iterates over +multiple configuration variations, and uses external projects to do +light integration testing to verify the keystone API against other projects. -You can enable a huge amount of additional data (debugging information) about -the request and repsonse objects flowing through Keystone using the debugging -WSGI middleware. +Test Structure +-------------- -To enable this, just modify the pipelines in ``etc/keystone.conf``, from:: +UPDATE THIS... - [pipeline:admin] - pipeline = - urlnormalizer - admin_api +Testing Schema Migrations +------------------------- - [pipeline:keystone-legacy-auth] - pipeline = - urlnormalizer - legacy_auth - d5_compat - service_api +The application of schema migrations can be tested using SQLAlchemy Migrate’s built-in test runner, one migration at a time. -... to:: +.. WARNING:: - [pipeline:admin] - pipeline = - debug - urlnormalizer - d5_compat - admin_api + This may leave your database in an inconsistent state; attempt this in non-production environments only! - [pipeline:keystone-legacy-auth] - pipeline = - debug - urlnormalizer - legacy_auth - d5_compat - service_api +This is useful for testing the *next* migration in sequence (both forward & backward) in a database under version control:: -Two simple and easy debugging tools are using the ``-d`` when you start keystone:: + python keystone/common/sql/migrate_repo/manage.py test \ + --url=sqlite:///test.db \ + --repository=keystone/common/sql/migrate_repo/ - $ ./keystone -d +This command references to a SQLite database (test.db) to be used. Depending on the migration, this command alone does not make assertions as to the integrity of your data during migration. -and the `--trace-calls` flag:: +Writing Tests +------------- - $ ./keystone -trace-calls +UPDATE THIS... -The ``-d`` flag outputs debug information to the console. The ``--trace-calls`` flag -outputs extensive, nested trace calls to the console and highlights any errors -in red. +Further Testing +--------------- + +devstack_ is the *best* way to quickly deploy keystone with the rest of the +OpenStack universe and should be critical step in your development workflow! + +You may also be interested in either the `OpenStack Continuous Integration Project`_ +or the `OpenStack Integration Testing Project`_. + +.. _devstack: http://devstack.org/ +.. _OpenStack Continuous Integration Project: https://github.com/openstack/openstack-ci +.. _OpenStack Integration Testing Project: https://github.com/openstack/tempest diff --git a/docs/source/index.rst b/docs/source/index.rst index 4270033a80..427414d475 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,9 +18,9 @@ Welcome to Keystone, the OpenStack Identity Service! ==================================================== -Keystone is a cloud identity service written in Python, which provides -authentication, authorization, and an OpenStack service catalog. It -implements `OpenStack's Identity API`_. +Keystone is an OpenStack project that provides Identity, Token, Catalog and +Policy services for use specifically by projects in the OpenStack family. +It implements `OpenStack's Identity API`_. This document describes Keystone for contributors of the project, and assumes that you are already familiar with Keystone from an `end-user perspective`_. @@ -42,9 +42,18 @@ Getting Started :maxdepth: 1 setup + configuration + configuringservices community - testing +Man Pages +--------- + +.. toctree:: + :maxdepth: 1 + + man/keystone + man/keystone-manage Developers Documentation ======================== @@ -53,7 +62,8 @@ Developers Documentation developing architecture - sourcecode/autoindex + api_curl_examples + modules Indices and tables ================== diff --git a/docs/source/man/keystone-manage.rst b/docs/source/man/keystone-manage.rst index 9e9304f04d..da5fc94152 100644 --- a/docs/source/man/keystone-manage.rst +++ b/docs/source/man/keystone-manage.rst @@ -22,159 +22,202 @@ DESCRIPTION =========== keystone-manage is the command line tool that interacts with the keystone -service to configure Keystone +service to initialize and update data within Keystone. Keystone *must* be +opertional for the keystone-manage commands to function correctly. USAGE ===== ``keystone-manage [options] type action [additional args]`` -user ----- -* **user add** [username] [password] +General keystone-manage options: +-------------------------------- - adds a user to Keystone's data store +* ``--id-only`` : causes ``keystone-manage`` to return only the UUID result +from the API call. +* ``--endpoint`` : allows you to specify the keystone endpoint to communicate with. The default endpoint is http://localhost:35357/v2.0' +* ``--auth-token`` : provides the authorization token -* **user list** +``keystone-manage`` is set up to expect commands in the general form of ``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to provide additional information to the command. For example, the command +``tenant`` has the subcommand ``create``, which takes the required keyword ``tenant_name``:: - lists all users + keystone-manage tenant create tenant_name=example_tenant -* **user disable** [username] +Invoking keystone-manage by itself will give you some usage information. - disables the user *username* +Available keystone-manage commands: + db_sync: Sync the database. + ec2: no docs + role: Role CRUD functions. + service: Service CRUD functions. + tenant: Tenant CRUD functions. + token: Token CRUD functions. + user: User CRUD functions. -tenant ------- - -* **tenant add** [tenant_name] - - adds a tenant to Keystone's data store - -* **tenant list** - - lists all users - -* **tenant disable** [tenant_name] - -role ----- - -Roles are used to associated users to tenants. Two roles are defined related -to the Keystone service in it's configuration file :doc:`../keystone.conf` - -* **role add** [role_name] - - adds a role - -* **role list** ([tenant_name]) - - lists all roles, or all roles for tenant, if tenant_name is provided - -* **role grant** [role_name] [username] ([tenant]) - - grants a role to a specific user. Granted globally if tenant_name is not - provided or granted for a specific tenant if tenant_name is provided. - -service +Tenants ------- -* **service add** [name] [type] [description] [owner_id] +Tenants are the high level grouping within Keystone that represent groups of +users. A tenant is the grouping that owns virtual machines within Nova, or +containers within Swift. A tenant can have zero or more users, Users can be assocaited with more than one tenant, and each tenant - user pairing can have a role associated with it. - adds a service +* tenant create -* **service list** + keyword arguments + * tenant_name + * id (optional) - lists all services with id, name, and type +example:: + keystone-manage --id-only tenant create tenant_name=admin -endpointTemplate ----------------- +creates a tenant named "admin". -* **endpointTemplate add** [region] [service_name] [public_url] [admin_url] [internal_url] [enabled] [is_global] +* tenant delete - Add a service endpoint for keystone. + keyword arguments + * tenant_id + +example:: + keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 - example:: +* tenant update - keystone-manage endpointTemplates add RegionOne \ - keystone \ - http://keystone_host:5000/v2.0 \ - http://keystone_host:35357/v2.0 \ - http://keystone_host:5000/v2.0 \ - 1 1 + keyword arguments + * description + * name + * tenant_id -* **endpointTemplate list** ([tenant_name]) +example:: + keystone-manage tenant update \ + tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 \ + description="those other guys" \ + name=tog - lists endpoint templates with service, region, and public_url. Restricted to - tenant endpoints if tenant_name is provided. - -token +Users ----- -* **token add** [token] [username] [tenant] [expiration] +* user create - adds a token for a given user and tenant with an expiration + keyword arguments + * name + * password + * email + +example:: + keystone-manage user --ks-id-only create \ + name=admin \ + password=secrete \ + email=admin@example.com + +* user delete -* **token list** + keyword arguments - lists all tokens +* user list -* **token delete** [token] + keyword arguments - deletes the identified token +* user update_email -endpoint + keyword arguments + +* user update_enabled + + keyword arguments + +* user update_password + + keyword arguments + +* user update_tenant + + keyword arguments + +Roles +----- + +* role create + + keyword arguments + * name + +exmaple:: + keystone-manage role --ks-id-only create name=Admin + +* role add_user_to_tenant + + keyword arguments + * role_id + * user_id + * tenant_id + +example:: + + keystone-manage role add_user_to_tenant \ + role_id=19d1d3344873464d819c45f521ff9890 \ + user_id=08741d8ed88242ca88d1f61484a0fe3b \ + tenant_id=20601a7f1d94447daa4dff438cb1c209 + +* role remove_user_from_tenant + +* role get_user_role_refs + +Services -------- -* **endpoint add** [tenant_name] [endpoint_template] +* service create - adds a tenant-specific endpoint + keyword arguments + * name + * service_type + * description -credentials ------------ +example:: + keystone-manage service create \ + name=nova \ + service_type=compute \ + description="Nova Compute Service" -* **credentials add** [username] [type] [key] [password] ([tenant_name]) OPTIONS ======= - --version show program's version number and exit - -h, --help show this help message and exit - -v, --verbose Print more verbose output - -d, --debug Print debugging output to console - -c PATH, --config-file=PATH Path to the config file to use. When not - specified (the default), we generally look at - the first argument specified to be a config - file, and if that is also missing, we search - standard directories for a config file. - -p BIND_PORT, --port=BIND_PORT, --bind-port=BIND_PORT - specifies port to listen on (default is 5000) - --host=BIND_HOST, --bind-host=BIND_HOST - specifies host address to listen on (default - is all or 0.0.0.0) - -t, --trace-calls Turns on call tracing for troubleshooting - -a PORT, --admin-port=PORT Specifies port for Admin API to listen on - (default is 35357) - -Logging Options: -================ - -The following configuration options are specific to logging -functionality for this program. - - --log-config=PATH If this option is specified, the logging - configuration file specified is used and - overrides any other logging options specified. - Please see the Python logging module - documentation for details on logging - configuration files. - --log-date-format=FORMAT Format string for %(asctime)s in log records. - Default: %Y-%m-%d %H:%M:%S - --log-file=PATH (Optional) Name of log file to output to. If - not set, logging will go to stdout. - --log-dir=LOG_DIR (Optional) The directory to keep log files in - (will be prepended to --logfile) - +Options: + -h, --help show this help message and exit + --config-file=PATH Path to a config file to use. Multiple config files + can be specified, with values in later files taking + precedence. The default files used are: [] + -d, --debug Print debugging output + --nodebug Print debugging output + -v, --verbose Print more verbose output + --noverbose Print more verbose output + --log-config=PATH If this option is specified, the logging configuration + file specified is used and overrides any other logging + options specified. Please see the Python logging + module documentation for details on logging + configuration files. + --log-format=FORMAT A logging.Formatter log message format string which + may use any of the available logging.LogRecord + attributes. Default: none + --log-date-format=DATE_FORMAT + Format string for %(asctime)s in log records. Default: + none + --log-file=PATH (Optional) Name of log file to output to. If not set, + logging will go to stdout. + --log-dir=LOG_DIR (Optional) The directory to keep log files in (will be + prepended to --logfile) + --syslog-log-facility=SYSLOG_LOG_FACILITY + (Optional) The syslog facility to use when logging to + syslog (defaults to LOG_USER) + --use-syslog Use syslog for logging. + --nouse-syslog Use syslog for logging. + --endpoint=ENDPOINT + --auth-token=AUTH_TOKEN + authorization token + --id-only + --noid-only + FILES ===== diff --git a/docs/source/man/keystone.rst b/docs/source/man/keystone.rst index 48e062e436..74d0bb69e1 100644 --- a/docs/source/man/keystone.rst +++ b/docs/source/man/keystone.rst @@ -21,9 +21,7 @@ SYNOPSIS DESCRIPTION =========== -keystone starts both the service and administrative API servers for Keystone. -Use :doc:`keystone-control` to stop/start/restart and manage those services -once started. +keystone starts both the service and administrative APIs for Keystone. USAGE ===== @@ -32,47 +30,40 @@ USAGE Common Options: ^^^^^^^^^^^^^^^ - --version show program's version number and exit -h, --help show this help message and exit The following configuration options are common to all keystone programs.:: - -v, --verbose Print more verbose output - -d, --debug Print debugging output to console - -c PATH, --config-file=PATH Path to the config file to use. When not - specified (the default), we generally look at - the first argument specified to be a config - file, and if that is also missing, we search - standard directories for a config file. - -p BIND_PORT, --port=BIND_PORT, --bind-port=BIND_PORT - specifies port to listen on (default is 5000) - --host=BIND_HOST, --bind-host=BIND_HOST - specifies host address to listen on (default - is all or 0.0.0.0) - -t, --trace-calls Turns on call tracing for troubleshooting - -a PORT, --admin-port=PORT Specifies port for Admin API to listen on - (default is 35357) - -Logging Options: -^^^^^^^^^^^^^^^^ - -The following configuration options are specific to logging -functionality for this program.:: - - --log-config=PATH If this option is specified, the logging - configuration file specified is used and - overrides any other logging options specified. - Please see the Python logging module - documentation for details on logging - configuration files. - --log-date-format=FORMAT Format string for %(asctime)s in log records. - Default: %Y-%m-%d %H:%M:%S - --log-file=PATH (Optional) Name of log file to output to. If - not set, logging will go to stdout. - --log-dir=LOG_DIR (Optional) The directory to keep log files in - (will be prepended to --logfile) - + -h, --help show this help message and exit + --config-file=PATH Path to a config file to use. Multiple config files + can be specified, with values in later files taking + precedence. The default files used are: [] + -d, --debug Print debugging output + --nodebug Print debugging output + -v, --verbose Print more verbose output + --noverbose Print more verbose output + --log-config=PATH If this option is specified, the logging configuration + file specified is used and overrides any other logging + options specified. Please see the Python logging + module documentation for details on logging + configuration files. + --log-format=FORMAT A logging.Formatter log message format string which + may use any of the available logging.LogRecord + attributes. Default: none + --log-date-format=DATE_FORMAT + Format string for %(asctime)s in log records. Default: + none + --log-file=PATH (Optional) Name of log file to output to. If not set, + logging will go to stdout. + --log-dir=LOG_DIR (Optional) The directory to keep log files in (will be + prepended to --logfile) + --syslog-log-facility=SYSLOG_LOG_FACILITY + (Optional) The syslog facility to use when logging to + syslog (defaults to LOG_USER) + --use-syslog Use syslog for logging. + --nouse-syslog Use syslog for logging. + FILES ===== diff --git a/docs/source/setup.rst b/docs/source/setup.rst index e608f09e4c..25a43655e6 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -18,12 +18,12 @@ Setting up a Keystone development environment ============================================= -This document describes setting up keystone directly from GitHub_ +This document describes getting the source from keystone's `GitHub repository`_ for development purposes. To install keystone from packaging, refer instead to Keystone's `User Documentation`_. -.. _GitHub: http://github.com/openstack/keystone +.. _`GitHub Repository`: http://github.com/openstack/keystone .. _`User Documentation`: http://docs.openstack.org/ Prerequisites @@ -51,7 +51,7 @@ different version of the above, please document your configuration here! Getting the latest code ======================= -You can clone our latest code from our `Github repository`:: +Make a clone of the code from our `Github repository`:: $ git clone https://github.com/openstack/keystone.git @@ -59,13 +59,17 @@ When that is complete, you can:: $ cd keystone -.. _`Github repository`: https://github.com/openstack/keystone - Installing dependencies ======================= -Keystone maintains a list of PyPi_ dependencies, designed for use by -pip_. +Keystone maintains two lists of dependencies: + + tools/pip-requires + tools/pip-requires-test + +The first is the list of dependencies needed for running keystone, the second list includes dependencies used for active development and testing of keystone itself. + +These depdendencies can be installed from PyPi_ using the python tool pip_. .. _PyPi: http://pypi.python.org/ .. _pip: http://pypi.python.org/pypi/pip @@ -89,29 +93,51 @@ Mac OS X Lion (requires MacPorts_):: .. _MacPorts: http://www.macports.org/ -PyPi Packages -------------- +PyPi Packages and VirtualEnv +---------------------------- -Assuming you have any necessary binary packages & header files available -on your system, you can then install PyPi dependencies. +We recommend establishing a virtualenv to run keystone within. Virtualenv limits the python environment +to just what you're installing as depdendencies, useful to keep a clean environment for working on +Keystone. The tools directory in keystone has a script already created to make this very simple:: -You may also need to prefix `pip install` with `sudo`, depending on your -environment:: + $ python tools/install_venv.py - # Describe dependencies (including non-PyPi dependencies) - $ cat tools/pip-requires +This will create a local virtual environment in the directory ``.venv``. +Once created, you can activate this virtualenv for your current shell using:: - # Install all PyPi dependencies (for production, testing, and development) + $ source .venv/bin/activate + +The virtual environment can be disabled using the command:: + + $ deactivate + +You can also use ``tools\with_venv.sh`` to prefix commands so that they run +within the virtual environment. For more information on virtual environments, +see virtualenv_. + +.. _virtualenv: http://www.virtualenv.org/ + +If you want to run keystone outside of a virtualenv, you can install the dependencies directly +into your system from the requires files:: + + # Install the dependencies for running keystone $ pip install -r tools/pip-requires -Updating your PYTHONPATH -======================== - -There are a number of methods for getting Keystone into your PYTHON PATH, -the easiest of which is:: - + # Install the dependencies for developing, testing, and running keystone + $ pip install -r tools/pip-requires-test + # Fake-install the project by symlinking Keystone into your Python site-packages $ python setup.py develop + + +Verifying Keystone is set up +============================ + +Once set up, either directly or within a virtualenv, you should be able to invoke python and import +the libraries. If you're using a virtualenv, don't forget to activate it:: + + $ source .venv/bin/activate + $ python You should then be able to `import keystone` from your Python shell without issue:: @@ -124,8 +150,7 @@ If you want to check the version of Keystone you are running: >>> print keystone.version.version() 2012.1-dev - -If you can import keystone successfully, you should be ready to move on to :doc:`testing`. +If you can import keystone successfully, you should be ready to move on to :doc:`testing` and :doc:`developing` Troubleshooting =============== diff --git a/docs/source/testing.rst b/docs/source/testing.rst deleted file mode 100644 index 82a3360460..0000000000 --- a/docs/source/testing.rst +++ /dev/null @@ -1,77 +0,0 @@ -================ -Testing Keystone -================ - -Keystone uses a number of testing methodologies to ensure correctness. - -Running Built-In Tests -====================== - -To run the full suites of tests maintained within Keystone, run:: - - $ ./run_tests.sh --with-progress - -This shows realtime feedback during test execution, and iterates over -multiple configuration variations. - -This differs from how tests are executed from the continuous integration -environment. Specifically, Jenkins doesn't care about realtime progress, -and aborts after the first test failure (a fail-fast behavior):: - - $ ./run_tests.sh - -Testing Schema Migrations -========================= - -The application of schema migrations can be tested using SQLAlchemy Migrate’s built-in test runner, one migration at a time. - -.. WARNING:: - - This may leave your database in an inconsistent state; attempt this in non-production environments only! - -This is useful for testing the *next* migration in sequence (both forward & backward) in a database under version control:: - - $ python keystone/backends/sqlalchemy/migrate_repo/manage.py test --url=sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/ - -This command refers to a SQLite database used for testing purposes. Depending on the migration, this command alone does not make assertions as to the integrity of your data during migration. - -Writing Tests -============= - -Tests are maintained in the ``keystone.test`` module. Unit tests are -isolated from functional tests. - -Functional Tests ----------------- - -The ``keystone.test.functional.common`` module provides a ``unittest``-based -``httplib`` client which you can extend and use for your own tests. -Generally, functional tests should serve to illustrate intended use cases -and API behaviors. To help make your tests easier to read, the test client: - -- Authenticates with a known user name and password combination -- Asserts 2xx HTTP status codes (unless told otherwise) -- Abstracts keystone REST verbs & resources into single function calls - -Testing Multiple Configurations -------------------------------- - -Several variations of the default configuration are iterated over to -ensure test coverage of mutually exclusive featuresets, such as the -various backend options. - -These configuration templates are maintained in ``keystone/test/etc`` and -are iterated over by ``run_tests.py``. - -Further Testing -=============== - -devstack_ is the *best* way to quickly deploy keystone with the rest of the -OpenStack universe and should be critical step in your development workflow! - -You may also be interested in either the `OpenStack Continuous Integration Project`_ -or the `OpenStack Integration Testing Project`_. - -.. _devstack: http://devstack.org/ -.. _OpenStack Continuous Integration Project: https://github.com/openstack/openstack-ci -.. _OpenStack Integration Testing Project: https://github.com/openstack/openstack-integration-tests diff --git a/keystone/cli.py b/keystone/cli.py index 0469ab3a65..93e3620564 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -24,7 +24,7 @@ config.register_cli_str('endpoint', config.register_cli_str('auth-token', default='$admin_token', #group='ks', - help='asdasd', + help='authorization token', conf=CONF) config.register_cli_bool('id-only', default=False, diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 389c9bf0f8..305a7d4446 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -1,6 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 import json +import webob from keystone import config from keystone.common import wsgi @@ -98,3 +99,40 @@ class JsonBodyMiddleware(wsgi.Middleware): params[k] = v request.environ[PARAMS_ENV] = params + + +class Debug(wsgi.Middleware): + """ + Middleware that produces stream debugging traces to the console (stdout) + for HTTP requests and responses flowing through it. + """ + + @webob.dec.wsgify + def __call__(self, req): + print ("*" * 40) + " REQUEST ENVIRON" + for key, value in req.environ.items(): + print key, "=", value + print + resp = req.get_response(self.application) + + print ("*" * 40) + " RESPONSE HEADERS" + for (key, value) in resp.headers.iteritems(): + print key, "=", value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """ + Iterator that prints the contents of a wrapper string iterator + when iterated. + """ + print ("*" * 40) + " BODY" + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print From fec7598a07b4beca78fe31a549e5d3beb6c0dc5d Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 13:24:38 -0800 Subject: [PATCH 262/334] shifting older docs into old/ directory --- docs/source/adminAPI_curl_examples.rst | 387 ------------------ docs/source/configuration.rst | 2 +- docs/source/index.rst | 6 +- docs/source/{ => old}/configuringservices.rst | 0 docs/source/{ => old}/controllingservers.rst | 0 docs/source/{ => old}/endpoints.rst | 0 docs/source/{ => old}/extensions.rst | 0 docs/source/{ => old}/middleware.rst | 0 .../{ => old}/middleware_architecture.rst | 0 docs/source/{ => old}/migration.rst | 0 docs/source/{ => old}/nova-api-paste.rst | 0 docs/source/{ => old}/releases.rst | 0 docs/source/{ => old}/services.rst | 0 docs/source/{ => old}/ssl.rst | 0 docs/source/serviceAPI_curl_examples.rst | 69 ---- docs/source/usingkeystone.rst | 28 -- 16 files changed, 4 insertions(+), 488 deletions(-) delete mode 100644 docs/source/adminAPI_curl_examples.rst rename docs/source/{ => old}/configuringservices.rst (100%) rename docs/source/{ => old}/controllingservers.rst (100%) rename docs/source/{ => old}/endpoints.rst (100%) rename docs/source/{ => old}/extensions.rst (100%) rename docs/source/{ => old}/middleware.rst (100%) rename docs/source/{ => old}/middleware_architecture.rst (100%) rename docs/source/{ => old}/migration.rst (100%) rename docs/source/{ => old}/nova-api-paste.rst (100%) rename docs/source/{ => old}/releases.rst (100%) rename docs/source/{ => old}/services.rst (100%) rename docs/source/{ => old}/ssl.rst (100%) delete mode 100644 docs/source/serviceAPI_curl_examples.rst delete mode 100644 docs/source/usingkeystone.rst diff --git a/docs/source/adminAPI_curl_examples.rst b/docs/source/adminAPI_curl_examples.rst deleted file mode 100644 index 81f96c36d3..0000000000 --- a/docs/source/adminAPI_curl_examples.rst +++ /dev/null @@ -1,387 +0,0 @@ -.. - Copyright 2011 OpenStack, LLC - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -============================= -Admin API Examples Using Curl -============================= - -These examples assume a default port value of 35357, and depend on the -``sampledata`` bundled with keystone. - -GET / -===== - -Disover API version information, links to documentation (PDF, HTML, WADL), -and supported media types:: - - $ curl http://0.0.0.0:35357 - -or:: - - $ curl http://0.0.0.0:35357/v2.0/ - -Returns:: - - { - "version":{ - "id":"v2.0", - "status":"beta", - "updated":"2011-11-19T00:00:00Z", - "links":[ - { - "rel":"self", - "href":"http://127.0.0.1:35357/v2.0/" - }, - { - "rel":"describedby", - "type":"text/html", - "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/content/" - }, - { - "rel":"describedby", - "type":"application/pdf", - "href":"http://docs.openstack.org/api/openstack-identity-service/2.0/identity-dev-guide-2.0.pdf" - }, - { - "rel":"describedby", - "type":"application/vnd.sun.wadl+xml", - "href":"http://127.0.0.1:35357/v2.0/identity-admin.wadl" - } - ], - "media-types":[ - { - "base":"application/xml", - "type":"application/vnd.openstack.identity-v2.0+xml" - }, - { - "base":"application/json", - "type":"application/vnd.openstack.identity-v2.0+json" - } - ] - } - } - -GET /extensions -=============== - -Discover the API extensions enabled at the endpoint:: - - $ curl http://0.0.0.0:35357/extensions - -Returns:: - - { - "extensions":{ - "values":[] - } - } - -POST /tokens -============ - -Authenticate by exchanging credentials for an access token:: - - $ curl -d '{"auth":{"passwordCredentials":{"username": "joeuser", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:35357/v2.0/tokens - -Returns:: - - { - "access":{ - "token":{ - "expires":"2012-02-05T00:00:00", - "id":"887665443383838", - "tenant":{ - "id":"1", - "name":"customer-x" - } - }, - "serviceCatalog":[ - { - "endpoints":[ - { - "adminURL":"http://swift.admin-nets.local:8080/", - "region":"RegionOne", - "internalURL":"http://127.0.0.1:8080/v1/AUTH_1", - "publicURL":"http://swift.publicinternets.com/v1/AUTH_1" - } - ], - "type":"object-store", - "name":"swift" - }, - { - "endpoints":[ - { - "adminURL":"http://cdn.admin-nets.local/v1.1/1", - "region":"RegionOne", - "internalURL":"http://127.0.0.1:7777/v1.1/1", - "publicURL":"http://cdn.publicinternets.com/v1.1/1" - } - ], - "type":"object-store", - "name":"cdn" - } - ], - "user":{ - "id":"1", - "roles":[ - { - "tenantId":"1", - "id":"3", - "name":"Member" - } - ], - "name":"joeuser" - } - } - } - -.. note:: - - Take note of the value ['access']['token']['id'] value produced here (``887665443383838``, above), as you can use it in the calls below. - -GET /tokens/{token_id} -====================== - -.. note:: - - This call refers to a token known to be valid, ``887665443383838`` in this case. - -Validate a token:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 - -If the token is valid, returns:: - - { - "access":{ - "token":{ - "expires":"2012-02-05T00:00:00", - "id":"887665443383838", - "tenant":{ - "id":"1", - "name":"customer-x" - } - }, - "user":{ - "name":"joeuser", - "tenantName":"customer-x", - "id":"1", - "roles":[ - { - "serviceId":"1", - "id":"3", - "name":"Member" - } - ], - "tenantId":"1" - } - } - } - -HEAD /tokens/{token_id} -======================= - -This is a high-performance variant of the GET call documented above, which -by definition, returns no response body:: - - $ curl -I -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838 - -... which returns ``200``, indicating the token is valid:: - - HTTP/1.1 200 OK - Content-Length: 0 - Content-Type: None - Date: Tue, 08 Nov 2011 23:07:44 GMT - -GET /tokens/{token_id}/endpoints -================================ - -List all endpoints for a token:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tokens/887665443383838/endpoints - -Returns:: - - { - "endpoints_links": [ - { - "href": "http://127.0.0.1:35357/tokens/887665443383838/endpoints?'marker=5&limit=10'", - "rel": "next" - } - ], - "endpoints": [ - { - "internalURL": "http://127.0.0.1:8080/v1/AUTH_1", - "name": "swift", - "adminURL": "http://swift.admin-nets.local:8080/", - "region": "RegionOne", - "tenantId": 1, - "type": "object-store", - "id": 1, - "publicURL": "http://swift.publicinternets.com/v1/AUTH_1" - }, - { - "internalURL": "http://localhost:8774/v1.0", - "name": "nova_compat", - "adminURL": "http://127.0.0.1:8774/v1.0", - "region": "RegionOne", - "tenantId": 1, - "type": "compute", - "id": 2, - "publicURL": "http://nova.publicinternets.com/v1.0/" - }, - { - "internalURL": "http://localhost:8774/v1.1", - "name": "nova", - "adminURL": "http://127.0.0.1:8774/v1.1", - "region": "RegionOne", - "tenantId": 1, - "type": "compute", - "id": 3, - "publicURL": "http://nova.publicinternets.com/v1.1/ - }, - { - "internalURL": "http://127.0.0.1:9292/v1.1/", - "name": "glance", - "adminURL": "http://nova.admin-nets.local/v1.1/", - "region": "RegionOne", - "tenantId": 1, - "type": "image", - "id": 4, - "publicURL": "http://glance.publicinternets.com/v1.1/" - }, - { - "internalURL": "http://127.0.0.1:7777/v1.1/1", - "name": "cdn", - "adminURL": "http://cdn.admin-nets.local/v1.1/1", - "region": "RegionOne", - "tenantId": 1, - "versionId": "1.1", - "versionList": "http://127.0.0.1:7777/", - "versionInfo": "http://127.0.0.1:7777/v1.1", - "type": "object-store", - "id": 5, - "publicURL": "http://cdn.publicinternets.com/v1.1/1" - } - ] - } - -GET /tenants -============ - -List all of the tenants in the system (requires an Admin ``X-Auth-Token``):: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants - -Returns:: - - { - "tenants_links": [], - "tenants": [ - { - "enabled": false, - "description": "None", - "name": "project-y", - "id": "3" - }, - { - "enabled": true, - "description": "None", - "name": "ANOTHER:TENANT", - "id": "2" - }, - { - "enabled": true, - "description": "None", - "name": "customer-x", - "id": "1" - } - ] - } - -GET /tenants/{tenant_id} -======================== - -Retrieve information about a tenant, by tenant ID:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1 - -Returns:: - - { - "tenant":{ - "enabled":true, - "description":"None", - "name":"customer-x", - "id":"1" - } - } - -GET /tenants/{tenant_id}/users/{user_id}/roles -============================================== - -List the roles a user has been granted on a tenant:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/tenants/1/users/1/roles - -Returns:: - - { - "roles_links":[], - "roles":[ - { - "id":"3", - "name":"Member" - } - ] - } - -GET /users/{user_id} -==================== - -Retrieve information about a user, by user ID:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/1 - -Returns:: - - { - "user":{ - "tenantId":"1", - "enabled":true, - "id":"1", - "name":"joeuser" - } - } - -GET /users/{user_id}/roles -========================== - -Retrieve the roles granted to a user, given a user ID:: - - $ curl -H "X-Auth-Token:999888777666" http://localhost:35357/v2.0/users/4/roles - -Returns:: - - { - "roles_links":[], - "roles":[ - { - "id":"2", - "name":"KeystoneServiceAdmin" - } - ] - } diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 02fbd4b44f..e16d1892e8 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -24,7 +24,7 @@ Configuring Keystone man/keystone-manage Once Keystone is installed, it is configured via a primary configuration file -(:doc:`keystone.conf`), possibly a separate logging configuration file, and +(``etc/keystone.conf``), possibly a separate logging configuration file, and initializing data into keystone using the command line client. diff --git a/docs/source/index.rst b/docs/source/index.rst index 427414d475..8b12ff9315 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,8 +42,8 @@ Getting Started :maxdepth: 1 setup - configuration - configuringservices + configuration + configuringservices community Man Pages @@ -53,7 +53,7 @@ Man Pages :maxdepth: 1 man/keystone - man/keystone-manage + man/keystone-manage Developers Documentation ======================== diff --git a/docs/source/configuringservices.rst b/docs/source/old/configuringservices.rst similarity index 100% rename from docs/source/configuringservices.rst rename to docs/source/old/configuringservices.rst diff --git a/docs/source/controllingservers.rst b/docs/source/old/controllingservers.rst similarity index 100% rename from docs/source/controllingservers.rst rename to docs/source/old/controllingservers.rst diff --git a/docs/source/endpoints.rst b/docs/source/old/endpoints.rst similarity index 100% rename from docs/source/endpoints.rst rename to docs/source/old/endpoints.rst diff --git a/docs/source/extensions.rst b/docs/source/old/extensions.rst similarity index 100% rename from docs/source/extensions.rst rename to docs/source/old/extensions.rst diff --git a/docs/source/middleware.rst b/docs/source/old/middleware.rst similarity index 100% rename from docs/source/middleware.rst rename to docs/source/old/middleware.rst diff --git a/docs/source/middleware_architecture.rst b/docs/source/old/middleware_architecture.rst similarity index 100% rename from docs/source/middleware_architecture.rst rename to docs/source/old/middleware_architecture.rst diff --git a/docs/source/migration.rst b/docs/source/old/migration.rst similarity index 100% rename from docs/source/migration.rst rename to docs/source/old/migration.rst diff --git a/docs/source/nova-api-paste.rst b/docs/source/old/nova-api-paste.rst similarity index 100% rename from docs/source/nova-api-paste.rst rename to docs/source/old/nova-api-paste.rst diff --git a/docs/source/releases.rst b/docs/source/old/releases.rst similarity index 100% rename from docs/source/releases.rst rename to docs/source/old/releases.rst diff --git a/docs/source/services.rst b/docs/source/old/services.rst similarity index 100% rename from docs/source/services.rst rename to docs/source/old/services.rst diff --git a/docs/source/ssl.rst b/docs/source/old/ssl.rst similarity index 100% rename from docs/source/ssl.rst rename to docs/source/old/ssl.rst diff --git a/docs/source/serviceAPI_curl_examples.rst b/docs/source/serviceAPI_curl_examples.rst deleted file mode 100644 index d05afc9ff3..0000000000 --- a/docs/source/serviceAPI_curl_examples.rst +++ /dev/null @@ -1,69 +0,0 @@ -.. - Copyright 2011 OpenStack, LLC - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=============================== -Service API Examples Using Curl -=============================== - -The service API is defined to be a subset of the Admin API and, by -default, runs on port 5000. - -GET / -===== - -This call is identical to that documented for the Admin API, except -that it uses port 5000, instead of port 35357, by default:: - - $ curl http://0.0.0.0:5000 - -or:: - - $ curl http://0.0.0.0:5000/v2.0/ - -See the `Admin API Examples Using Curl`_ for more info. - -.. _`Admin API Examples Using Curl`: adminAPI_curl_examples.html - -GET /extensions -=============== - -This call is identical to that documented for the Admin API. - -POST /tokens -============ - -This call is identical to that documented for the Admin API. - -GET /tenants -============ - -List all of the tenants your token can access:: - - $ curl -H "X-Auth-Token:887665443383838" http://localhost:5000/v2.0/tenants - -Returns:: - - { - "tenants_links": [], - "tenants": [ - { - "enabled": true, - "description": "None", - "name": "customer-x", - "id": "1" - } - ] - } diff --git a/docs/source/usingkeystone.rst b/docs/source/usingkeystone.rst deleted file mode 100644 index bb52a94d09..0000000000 --- a/docs/source/usingkeystone.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. - Copyright 2011 OpenStack, LLC - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -============== -Using Keystone -============== - -Curl examples -------------- - -.. toctree:: - :maxdepth: 1 - - adminAPI_curl_examples - serviceAPI_curl_examples From 1908a2d7ba14b830648078e6871fba0e6644fdf5 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 13:47:29 -0800 Subject: [PATCH 263/334] format tweaks and moving old docs --- docs/source/configuringservices.rst | 174 +++++++++ docs/source/index.rst | 6 + .../{old => }/middleware_architecture.rst | 0 docs/source/{old => }/nova-api-paste.rst | 0 docs/source/{ => old}/backends.rst | 0 docs/source/old/configuringservices.rst | 333 ------------------ docs/source/setup.rst | 2 +- 7 files changed, 181 insertions(+), 334 deletions(-) create mode 100644 docs/source/configuringservices.rst rename docs/source/{old => }/middleware_architecture.rst (100%) rename docs/source/{old => }/nova-api-paste.rst (100%) rename docs/source/{ => old}/backends.rst (100%) delete mode 100644 docs/source/old/configuringservices.rst diff --git a/docs/source/configuringservices.rst b/docs/source/configuringservices.rst new file mode 100644 index 0000000000..88bb9c15db --- /dev/null +++ b/docs/source/configuringservices.rst @@ -0,0 +1,174 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +========================================== +Configuring Services to work with Keystone +========================================== + +.. toctree:: + :maxdepth: 1 + + nova-api-paste + middleware_architecture + +Once Keystone is installed and running (see :doc:`configuration`), services need to be configured to work +with it. To do this, we primarily install and configure middleware for the OpenStack service to handle authentication tasks or otherwise interact with Keystone. + +In general: +* Clients making calls to the service will pass in an authentication token. +* The Keystone middleware will look for and validate that token, taking the appropriate action. +* It will also retrive additional information from the token such as user name, id, tenant name, id, roles, etc... + +The middleware will pass those data down to the service as headers. More details on the architecture of +that setup is described in :doc:`middleware_architecture` + +Setting up credentials +====================== + +Admin Token +----------- + +For a default installation of Keystone, before you can use the REST API, you +need to define an authorization token. This is configured in the keystone.conf file under the section ``[DEFAULT]``. In the sample file provided with the keystone project, the line defining this token is + + [DEFAULT] + admin_token = ADMIN + +This is a "shared secret" between keystone and other openstack services, and will need to be set the +same between those services in order for keystone services to function correctly. + +Setting up tenants, users, and roles +------------------------------------ + +You need to minimally define a tenant, user, and role to link the tenant and user as the most basic set of details to get other services authenticating and authorizing with keystone. See doc:`configuration` for a walk through on how to create tenants, users, and roles. + +Setting up services +=================== + +Defining Services +----------------- + +Keystone also acts as a service catalog to let other OpenStack systems know +where relevant API endpoints exist for OpenStack Services. The OpenStack +Dashboard, in particular, uses this heavily - and this **must** be configured +for the OpenStack Dashboard to properly function. + +Here's how we define the services:: + + keystone-manage service create name=nova service_type=compute description="Nova Compute Service" + keystone-manage service create name=ec2 service_type=ec2 description="EC2 Compatibility Layer" + keystone-manage service create name=glance service_type=image description="Glance Image Service" + keystone-manage service create name=keystone service_type=identity description="Keystone Identity Service" + keystone-manage service create name=swift service_type=object-store description="Swift Service" + +The endpoints for these services are defined in a template, an example of which is in the project as the file ``etc/default_catalog.templates``. + +Setting Up Middleware +===================== + +Keystone Auth-Token Middleware +-------------------------------- + +The Keystone auth_token middleware is a WSGI component that can be inserted in +the WSGI pipeline to handle authenticating tokens with Keystone. + +Configuring Nova to use Keystone +-------------------------------- + +To configure Nova to use Keystone for authentication, the Nova API service +can be run against the api-paste file provided by Keystone. This is most +easily accomplished by setting the `--api_paste_config` flag in nova.conf to +point to `examples/paste/nova-api-paste.ini` from Keystone. This paste file +included references to the WSGI authentication middleware provided with the +keystone installation. + +When configuring Nova, it is important to create a admin service token for +the service (from the Configuration step above) and include that as the key +'admin_token' in the nova-api-paste.ini. See the documented +:doc:`nova-api-paste` file for references. + +Configuring Swift to use Keystone +--------------------------------- + +Similar to Nova, swift can be configured to use Keystone for authentication +rather than it's built in 'tempauth'. + +1. Add a service endpoint for Swift to Keystone + +2. Configure the paste file for swift-proxy (`/etc/swift/swift-proxy.conf`) + +3. Reconfigure Swift's proxy server to use Keystone instead of TempAuth. + Here's an example `/etc/swift/proxy-server.conf`:: + + [DEFAULT] + bind_port = 8888 + user = + + [pipeline:main] + pipeline = catch_errors cache keystone proxy-server + + [app:proxy-server] + use = egg:swift#proxy + account_autocreate = true + + [filter:keystone] + use = egg:keystone#tokenauth + auth_protocol = http + auth_host = 127.0.0.1 + auth_port = 35357 + admin_token = 999888777666 + delay_auth_decision = 0 + service_protocol = http + service_host = 127.0.0.1 + service_port = 8100 + service_pass = dTpw + cache = swift.cache + + [filter:cache] + use = egg:swift#memcache + set log_name = cache + + [filter:catch_errors] + use = egg:swift#catch_errors + + Note that the optional "cache" property in the keystone filter allows any + service (not just Swift) to register its memcache client in the WSGI + environment. If such a cache exists, Keystone middleware will utilize it + to store validated token information, which could result in better overall + performance. + +4. Restart swift + +5. Verify that keystone is providing authentication to Swift + +Use `swift` to check everything works (note: you currently have to create a +container or upload something as your first action to have the account +created; there's a Swift bug to be fixed soon):: + + $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete post container + $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete stat -v + StorageURL: http://127.0.0.1:8888/v1/AUTH_1234 + Auth Token: 74ce1b05-e839-43b7-bd76-85ef178726c3 + Account: AUTH_1234 + Containers: 1 + Objects: 0 + Bytes: 0 + Accept-Ranges: bytes + X-Trans-Id: tx25c1a6969d8f4372b63912f411de3c3b + +.. WARNING:: + Keystone currently allows any valid token to do anything with any account. + diff --git a/docs/source/index.rst b/docs/source/index.rst index 8b12ff9315..5d7c80faee 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -63,6 +63,12 @@ Developers Documentation developing architecture api_curl_examples + +Code Documentation +================== +.. toctree:: + :maxdepth: 1 + modules Indices and tables diff --git a/docs/source/old/middleware_architecture.rst b/docs/source/middleware_architecture.rst similarity index 100% rename from docs/source/old/middleware_architecture.rst rename to docs/source/middleware_architecture.rst diff --git a/docs/source/old/nova-api-paste.rst b/docs/source/nova-api-paste.rst similarity index 100% rename from docs/source/old/nova-api-paste.rst rename to docs/source/nova-api-paste.rst diff --git a/docs/source/backends.rst b/docs/source/old/backends.rst similarity index 100% rename from docs/source/backends.rst rename to docs/source/old/backends.rst diff --git a/docs/source/old/configuringservices.rst b/docs/source/old/configuringservices.rst deleted file mode 100644 index 083c3ec5ed..0000000000 --- a/docs/source/old/configuringservices.rst +++ /dev/null @@ -1,333 +0,0 @@ -.. - Copyright 2011 OpenStack, LLC - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -========================================== -Configuring Services to work with Keystone -========================================== - -.. toctree:: - :maxdepth: 1 - -Once Keystone is installed and running, services need to be configured to work -with it. These are the steps to configure a service to work with Keystone: - -1. Create or get credentials for the service to use - - A set of credentials are needed for each service (they may be - shared if you chose to). Depending on the service, these credentials are - either a username and password or a long-lived token.. - -2. Register the service, endpoints, roles and other entities - - In order for a service to have it's endpoints and roles show in the service - catalog returned by Keystone, a service record needs to be added for the - service. Endpoints and roles associated with that service can then be created. - - This can be done through the REST interface (using the OS-KSCATALOG extension) - or using keystone-manage. - -3. Install and configure middleware for the service to handle authentication - - Clients making calls to the service will pass in an authentication token. The - Keystone middleware will look for and validate that token, taking the - appropriate action. It will also retrive additional information from the token - such as user name, id, tenant name, id, roles, etc... - - The middleware will pass those data down to the service as headers. The - detailed description of this architecture is available here :doc:`middleware_architecture` - -Setting up credentials -====================== - -First admin user - bootstrapping --------------------------------- - -For a default installation of Keystone, before you can use the REST API, you -need to create your first initial user and grant that user the right to -administer Keystone. - -For the keystone service itself, two -Roles are pre-defined in the keystone configuration file -(:doc:`keystone.conf`). - - #Role that allows admin operations (access to all operations) - keystone-admin-role = Admin - - #Role that allows acting as service (validate tokens, register service, - etc...) - keystone-service-admin-role = KeystoneServiceAdmin - -In order to create your first user, once Keystone is running use -the `keystone-manage` command: - - $ keystone-manage user add admin secrete - $ keystone-manage role add Admin - $ keystone-manage role add KeystoneServiceAdmin - $ keystone-manage role grant Admin admin - $ keystone-manage role grant KeystoneServiceAdmin admin - -This creates the `admin` user (with a password of `secrete`), creates -two roles (`Admin` and `KeystoneServiceAdmin`), and assigns those roles to -the `admin` user. From here, you should now have the choice of using the -administrative API (as well as the :doc:`man/keystone-manage` commands) to -further configure keystone. There are a number of examples of how to use -that API at :doc:`adminAPI_curl_examples`. - - -Setting up services -=================== - -Defining Services and Service Endpoints ---------------------------------------- - -Keystone also acts as a service catalog to let other OpenStack systems know -where relevant API endpoints exist for OpenStack Services. The OpenStack -Dashboard, in particular, uses this heavily - and this **must** be configured -for the OpenStack Dashboard to properly function. - -Here's how we define the services:: - - $ keystone-manage service add nova compute "Nova Compute Service" - $ keystone-manage service add glance image "Glance Image Service" - $ keystone-manage service add swift storage "Swift Object Storage Service" - $ keystone-manage service add keystone identity "Keystone Identity Service" - -Once the services are defined, we create endpoints for them. Each service -has three relevant URL's associated with it that are used in the command: - -* the public API URL -* an administrative API URL -* an internal URL - -The "internal URL" is an endpoint the generally offers the same API as the -public URL, but over a high-bandwidth, low-latency, unmetered (free) network. -You would use that to transfer images from nova to glance for example, and -not the Public URL which would go over the internet and be potentially chargeable. - -The "admin URL" is for administering the services and is not exposed or accessible -to customers without the apporpriate privileges. - -An example of setting up the endpoint for Nova:: - - $ keystone-manage endpointTemplates add RegionOne nova \ - http://nova-api.mydomain:8774/v1.1/%tenant_id% \ - http://nova-api.mydomain:8774/v1.1/%tenant_id% \ - http://nova-api.mydomain:8774/v1.1/%tenant_id% \ - 1 1 - -Glance:: - - $ keystone-manage endpointTemplates add RegionOne glance \ - http://glance.mydomain:9292/v1 \ - http://glance.mydomain:9292/v1 \ - http://glance.mydomain:9292/v1 \ - 1 1 - -Swift:: - - $ keystone-manage endpointTemplates add RegionOne swift \ - http://swift.mydomain:8080/v1/AUTH_%tenant_id% \ - http://swift.mydomain:8080/v1.0/ \ - http://swift.mydomain:8080/v1/AUTH_%tenant_id% \ - 1 1 - -And setting up an endpoint for Keystone:: - - $ keystone-manage endpointTemplates add RegionOne keystone \ - http://keystone.mydomain:5000/v2.0 \ - http://keystone.mydomain:35357/v2.0 \ - http://keystone.mydomain:5000/v2.0 \ - 1 1 - - -Defining an Administrative Service Token ----------------------------------------- - -An Administrative Service Token is a bit of arbitrary text which is configured -in Keystone and used (typically configured into) Nova, Swift, Glance, and any -other OpenStack projects, to be able to use Keystone services. - -This token is an arbitrary text string, but must be identical between Keystone -and the services using Keystone. This token is bound to a user and tenant as -well, so those also need to be created prior to setting it up. - -The *admin* user was set up above, but we haven't created a tenant for that -user yet:: - - $ keystone-manage tenant add admin - -and while we're here, let's grant the admin user the 'Admin' role to the -'admin' tenant:: - - $ keystone-manage role add Admin - $ keystone-manage role grant Admin admin admin - -Now we can create a service token:: - - $ keystone-manage token add 999888777666 admin admin 2015-02-05T00:00 - -This creates a service token of '999888777666' associated to the admin user, -admin tenant, and expires on February 5th, 2015. This token will be used when -configuring Nova, Glance, or other OpenStack services. - -Securing Communications with SSL --------------------------------- - -To encrypt traffic between services and Keystone, see :doc:`ssl` - - -Setting up OpenStack users -========================== - -Creating Tenants, Users, and Roles ----------------------------------- - -Let's set up a 'demo' tenant:: - - $ keystone-manage tenant add demo - -And add a 'demo' user with the password 'guest':: - - $ keystone-manage user add demo guest - -Now let's add a role of "Member" and grant 'demo' user that role -as it pertains to the tenant 'demo':: - - $ keystone-manage role add Member - $ keystone-manage role grant Member demo demo - -Let's also add the admin user as an Admin role to the demo tenant:: - - $ keystone-manage role grant Admin admin demo - -Creating EC2 credentials ------------------------- - -To add EC2 credentials for the `admin` and `demo` accounts:: - - $ keystone-manage credentials add admin EC2 'admin' 'secretpassword' - $ keystone-manage credentials add admin EC2 'demo' 'secretpassword' - -If you have a large number of credentials to create, you can put them all -into a single large file and import them using :doc:`man/keystone-import`. The -format of the document looks like:: - - credentials add admin EC2 'username' 'password' - credentials add admin EC2 'username' 'password' - -Then use:: - - $ keystone-import `filename` - - -Setting Up Middleware -===================== - -Keystone Auth-Token Middleware --------------------------------- - -The Keystone auth_token middleware is a WSGI component that can be inserted in -the WSGI pipeline to handle authenticating tokens with Keystone. See :doc:`middleware` -for details on middleware and configuration parameters. - - -Configuring Nova to use Keystone --------------------------------- - -To configure Nova to use Keystone for authentication, the Nova API service -can be run against the api-paste file provided by Keystone. This is most -easily accomplished by setting the `--api_paste_config` flag in nova.conf to -point to `examples/paste/nova-api-paste.ini` from Keystone. This paste file -included references to the WSGI authentication middleware provided with the -keystone installation. - -When configuring Nova, it is important to create a admin service token for -the service (from the Configuration step above) and include that as the key -'admin_token' in the nova-api-paste.ini. See the documented -:doc:`nova-api-paste` file for references. - -Configuring Swift to use Keystone ---------------------------------- - -Similar to Nova, swift can be configured to use Keystone for authentication -rather than it's built in 'tempauth'. - -1. Add a service endpoint for Swift to Keystone - -2. Configure the paste file for swift-proxy (`/etc/swift/swift-proxy.conf`) - -3. Reconfigure Swift's proxy server to use Keystone instead of TempAuth. - Here's an example `/etc/swift/proxy-server.conf`:: - - [DEFAULT] - bind_port = 8888 - user = - - [pipeline:main] - pipeline = catch_errors cache keystone proxy-server - - [app:proxy-server] - use = egg:swift#proxy - account_autocreate = true - - [filter:keystone] - use = egg:keystone#tokenauth - auth_protocol = http - auth_host = 127.0.0.1 - auth_port = 35357 - admin_token = 999888777666 - delay_auth_decision = 0 - service_protocol = http - service_host = 127.0.0.1 - service_port = 8100 - service_pass = dTpw - cache = swift.cache - - [filter:cache] - use = egg:swift#memcache - set log_name = cache - - [filter:catch_errors] - use = egg:swift#catch_errors - - Note that the optional "cache" property in the keystone filter allows any - service (not just Swift) to register its memcache client in the WSGI - environment. If such a cache exists, Keystone middleware will utilize it - to store validated token information, which could result in better overall - performance. - -4. Restart swift - -5. Verify that keystone is providing authentication to Swift - -Use `swift` to check everything works (note: you currently have to create a -container or upload something as your first action to have the account -created; there's a Swift bug to be fixed soon):: - - $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete post container - $ swift -A http://127.0.0.1:5000/v1.0 -U joeuser -K secrete stat -v - StorageURL: http://127.0.0.1:8888/v1/AUTH_1234 - Auth Token: 74ce1b05-e839-43b7-bd76-85ef178726c3 - Account: AUTH_1234 - Containers: 1 - Objects: 0 - Bytes: 0 - Accept-Ranges: bytes - X-Trans-Id: tx25c1a6969d8f4372b63912f411de3c3b - -.. WARNING:: - Keystone currently allows any valid token to do anything with any account. - diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 25a43655e6..4f6b6137c6 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -150,7 +150,7 @@ If you want to check the version of Keystone you are running: >>> print keystone.version.version() 2012.1-dev -If you can import keystone successfully, you should be ready to move on to :doc:`testing` and :doc:`developing` +If you can import keystone successfully, you should be ready to move on to :doc:`developing` Troubleshooting =============== From ef8b8f1d9b61ff527b223d69c52f35f25536b8dd Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 14:09:24 -0800 Subject: [PATCH 264/334] updating formating for configuration page --- docs/source/configuration.rst | 133 +++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 52 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e16d1892e8..410e7b1bfe 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -101,13 +101,14 @@ from the API call. Invoking keystone-manage by itself will give you some usage information. Available keystone-manage commands: - db_sync: Sync the database. - ec2: no docs - role: Role CRUD functions. - service: Service CRUD functions. - tenant: Tenant CRUD functions. - token: Token CRUD functions. - user: User CRUD functions. + +* ``db_sync``: Sync the database. +* ``ec2``: no docs +* ``role``: Role CRUD functions. +* ``service``: Service CRUD functions. +* ``tenant``: Tenant CRUD functions. +* ``token``: Token CRUD functions. +* ``user``: User CRUD functions. Tenants ------- @@ -116,33 +117,42 @@ Tenants are the high level grouping within Keystone that represent groups of users. A tenant is the grouping that owns virtual machines within Nova, or containers within Swift. A tenant can have zero or more users, Users can be assocaited with more than one tenant, and each tenant - user pairing can have a role associated with it. -* tenant create +``tenant create`` +^^^^^^^^^^^^^^^^^ - keyword arguments - * tenant_name - * id (optional) +keyword arguments + +* tenant_name +* id (optional) example:: + keystone-manage --id-only tenant create tenant_name=admin creates a tenant named "admin". -* tenant delete +``tenant delete`` +^^^^^^^^^^^^^^^^^ - keyword arguments - * tenant_id +keyword arguments + +* tenant_id example:: + keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 -* tenant update +``tenant update`` +^^^^^^^^^^^^^^^^^ - keyword arguments - * description - * name - * tenant_id +keyword arguments + +* description +* name +* tenant_id example:: + keystone-manage tenant update \ tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 \ description="those other guys" \ @@ -151,60 +161,74 @@ example:: Users ----- -* user create +``user create`` +^^^^^^^^^^^^^^^ - keyword arguments - * name - * password - * email +keyword arguments + +* name +* password +* email example:: + keystone-manage user --ks-id-only create \ name=admin \ password=secrete \ email=admin@example.com -* user delete +``user delete`` +^^^^^^^^^^^^^^^ - keyword arguments +keyword arguments -* user list +``user list`` +^^^^^^^^^^^^^ - keyword arguments +keyword arguments -* user update_email +``user update_email`` +^^^^^^^^^^^^^^^^^^^^^ - keyword arguments +keyword arguments -* user update_enabled +``user update_enabled`` +^^^^^^^^^^^^^^^^^^^^^^^ - keyword arguments +keyword arguments -* user update_password +``user update_password`` +^^^^^^^^^^^^^^^^^^^^^^^^ - keyword arguments +keyword arguments -* user update_tenant +``user update_tenant`` +^^^^^^^^^^^^^^^^^^^^^^ - keyword arguments +keyword arguments Roles ----- -* role create +``role create`` +^^^^^^^^^^^^^^^ - keyword arguments - * name +keyword arguments + +* name exmaple:: + keystone-manage role --ks-id-only create name=Admin -* role add_user_to_tenant +``role add_user_to_tenant`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ - keyword arguments - * role_id - * user_id - * tenant_id +keyword arguments + +* role_id +* user_id +* tenant_id example:: @@ -213,22 +237,27 @@ example:: user_id=08741d8ed88242ca88d1f61484a0fe3b \ tenant_id=20601a7f1d94447daa4dff438cb1c209 -* role remove_user_from_tenant +``role remove_user_from_tenant`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* role get_user_role_refs +``role get_user_role_refs`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Services -------- -* service create +``service create`` +^^^^^^^^^^^^^^^^^^ - keyword arguments - * name - * service_type - * description +keyword arguments + +* name +* service_type +* description example:: - keystone-manage service create \ + + keystone-manage service create \ name=nova \ service_type=compute \ description="Nova Compute Service" From 22c3f8067ed7e480e5d06c67677a4f60d740a10d Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 14:14:09 -0800 Subject: [PATCH 265/334] moved notes from README.rst into docs/architecture.rst --- docs/source/architecture.rst | 224 +++++++++++++++++++++++++---------- docs/source/developing.rst | 14 +++ 2 files changed, 177 insertions(+), 61 deletions(-) diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index 21d9b895a5..b308a9e5ad 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -20,82 +20,184 @@ Keystone Architecture Much of the design is precipitated from the expectation that the auth backends for most deployments will actually be shims in front of existing user systems. -.........JOEEDIT.......... -Keystone has two major components: Authentication and a Service Catalog. +------------ +The Services +------------ -Authentication --------------- +Keystone is organized as a group of services exposed on one or many endpoints. +Many of these services are used in a combined fashion by the frontend, for +example an authenticate call will validate user/tenant credentials with the +Identity service and, upon success, create and return a token with the Token +service. -In providing a token-based authentication service for OpenStack, keystone -has several major concepts: -Tenant - A grouping used in OpenStack to contain relevant OpenStack services. A - tenant maps to a Nova "project-id", and in object storage, a tenant can - have multiple containers. Depending on the installation, a tenant can - represent a customer, account, organization, or project. +Identity +-------- -User - Represents an individual within OpenStack for the purposes of - authenticating them to OpenStack services. Users have credentials, and may - be assigned to one or more tenants. When authenticated, a token is - provided that is specific to a single tenant. +The Identity service provides auth credential validation and data about Users, +Tenants and Roles, as well as any associated metadata. + +In the basic case all this data is managed by the service, allowing the service +to manage all the CRUD associated with the data. + +In other cases, this data is pulled, by varying degrees, from an authoritative +backend service. An example of this would be when backending on LDAP. See +`LDAP Backend` below for more details. -Credentials - Password or other information that uniquely identifies a User to Keystone - for the purposes of providing a token. Token - A token is an arbitrary bit of text that is used to share authentication - with other OpenStack services so that Keystone can provide a central - location for authenticating users for access to OpenStack services. A - token may be "scoped" or "unscoped". A scoped token represents a user - authenticated to a Tenant, where an unscoped token represents just the - user. +----- - Tokens are valid for a limited amount of time and may be revoked at any - time. +The Token service validates and manages Tokens used for authenticating requests +once a user/tenant's credentials have already been verified. -Role - A role is a set of permissions to access and use specific operations for - a given user when applied to a tenant. Roles are logical groupings of - those permissions to enable common permissions to be easily grouped and - bound to users associated with a given tenant. -Service Catalog +Catalog +------- + +The Catalog service provides an endpoint registry used for endpoint discovery. + + +Policy +------ + +The Policy service provides a rule-based authorization engine and the +associated rule management interface. + +---------- +Data Model +---------- + +Keystone was designed from the ground up to be amenable to multiple styles of +backends and as such many of the methods and data types will happily accept +more data than they know what to do with and pass them on to a backend. + +There are a few main data types: + + * **User**: has account credentials, is associated with one or more tenants + * **Tenant**: unit of ownership in openstack, contains one or more users + * **Role**: a first-class piece of metadata associated with many user-tenant pairs. + * **Token**: identifying credential associated with a user or user and tenant + * **Extras**: bucket of key-value metadata associated with a user-tenant pair. + * **Rule**: describes a set of requirements for performing an action. + +While the general data model allows a many-to-many relationship between Users +and Tenants and a many-to-one relationship between Extras and User-Tenant pairs, +the actual backend implementations take varying levels of advantage of that +functionality. + + +KVS Backend +----------- + +A simple backend interface meant to be further backended on anything that can +support primary key lookups, the most trivial implementation being an in-memory +dict. + +Supports all features of the general data model. + + +PAM Backend +----------- + +Extra simple backend that uses the current system's PAM service to authenticate, +providing a one-to-one relationship between Users and Tenants with the `root` +User also having the 'admin' role. + + +Templated Backend +----------------- + +Largely designed for a common use case around service catalogs in the Keystone +project, a Catalog backend that simply expands pre-configured templates to +provide catalog data. + +Example paste.deploy config (uses $ instead of % to avoid ConfigParser's +interpolation):: + + [DEFAULT] + catalog.RegionOne.identity.publicURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.adminURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.internalURL = http://localhost:$(public_port)s/v2.0 + catalog.RegionOne.identity.name = 'Identity Service' + + +---------------- +Approach to CRUD +---------------- + +While it is expected that any "real" deployment at a large company will manage +their users, tenants and other metadata in their existing user systems, a +variety of CRUD operations are provided for the sake of development and testing. + +CRUD is treated as an extension or additional feature to the core feature set in +that it is not required that a backend support it. + + +---------------------------------- +Approach to Authorization (Policy) +---------------------------------- + +Various components in the system require that different actions are allowed +based on whether the user is authorized to perform that action. + +For the purposes of Keystone Light there are only a couple levels of +authorization being checked for: + + * Require that the performing user is considered an admin. + * Require that the performing user matches the user being referenced. + +Other systems wishing to use the policy engine will require additional styles +of checks and will possibly write completely custom backends. Backends included +in Keystone Light are: + + +Trivial True +------------ + +Allows all actions. + + +Simple Match +------------ + +Given a list of matches to check for, simply verify that the credentials +contain the matches. For example:: + + credentials = {'user_id': 'foo', 'is_admin': 1, 'roles': ['nova:netadmin']} + + # An admin only call: + policy_api.can_haz(('is_admin:1',), credentials) + + # An admin or owner call: + policy_api.can_haz(('is_admin:1', 'user_id:foo'), + credentials) + + # A netadmin call: + policy_api.can_haz(('roles:nova:netadmin',), + credentials) + + +Credentials are generally built from the user metadata in the 'extras' part +of the Identity API. So, adding a 'role' to the user just means adding the role +to the user metadata. + + +Capability RBAC --------------- -Keystone also provides a list of REST API endpoints as a definitive list for -an OpenStack installation. Key concepts include: +(Not yet implemented.) -Service - An OpenStack service such as nova, swift, glance, or keystone. A service - may have one of more endpoints through which users can interact with - OpenStack services and resources. +Another approach to authorization can be action-based, with a mapping of roles +to which capabilities are allowed for that role. For example:: -Endpoint - A network accessible address (typically a URL) that represents the API - interface to an OpenStack service. Endpoints may also be grouped into - templates which represent a group of consumable OpenStack services - available across regions. + credentials = {'user_id': 'foo', 'is_admin': 1, 'roles': ['nova:netadmin']} -Template - A collection of endpoints representing a set of consumable OpenStack - service endpoints. + # add a policy + policy_api.add_policy('action:nova:add_network', ('roles:nova:netadmin',)) -Components of Keystone ----------------------- + policy_api.can_haz(('action:nova:add_network',), credentials) -Keystone includes a command-line interface which interacts with the Keystone -API for administrating keystone and related services. -* keystone - runs both keystone-admin and keystone-service -* keystone-admin - the administrative API for manipulating keystone -* keystone-service - the user oriented API for authentication -* keystone-manage - the command line interface to manipulate keystone - -Keystone also includes WSGI middelware to provide authentication support -for Nova and Swift. - -Keystone uses a built-in SQLite datastore - and may use an external LDAP -service to authenticate users instead of using stored credentials. +In the backend this would look up the policy for 'action:nova:add_network' and +then do what is effectively a 'Simple Match' style match against the creds. diff --git a/docs/source/developing.rst b/docs/source/developing.rst index b19fec2455..48ba28734f 100644 --- a/docs/source/developing.rst +++ b/docs/source/developing.rst @@ -107,3 +107,17 @@ or the `OpenStack Integration Testing Project`_. .. _devstack: http://devstack.org/ .. _OpenStack Continuous Integration Project: https://github.com/openstack/openstack-ci .. _OpenStack Integration Testing Project: https://github.com/openstack/tempest + +Building the Documentation +========================== + +The documentation is all generated with Sphinx from within the docs directory. +To generate the full set of HTML documentation: + + cd docs + make autodoc + make html + make man + +the results are in the docs/build/html and docs/build/man directories +respectively. From d1f4ddcb81d470401f5b6c80b66ac9649e3c224e Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 29 Jan 2012 14:36:11 -0800 Subject: [PATCH 266/334] adding in testing details --- docs/source/developing.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/source/developing.rst b/docs/source/developing.rst index 48ba28734f..344a6e8bc0 100644 --- a/docs/source/developing.rst +++ b/docs/source/developing.rst @@ -70,7 +70,15 @@ light integration testing to verify the keystone API against other projects. Test Structure -------------- -UPDATE THIS... +``./run_test.sh`` uses it's python cohort (``run_tests.py``) to iterate through the ``tests`` directory, using Nosetest to collect the tests and invoke them using an +OpenStack custom test running that displays the tests as well as the time taken to +run those tests. + +Within the tests directory, the general structure of the tests is a basic set of tests represented under a test class, and then subclasses of those tests under other classes with different configurations to drive different backends through the APIs. + +For example, ``test_backend.py`` has a sequence of tests under the class ``IdentityTests`` that will work with the default drivers as configured in this projects etc/ directory. ``test_backend_sql.py`` subclasses those tests, changing the configuration by overriding with configuration files stored in the tests directory aimed at enabling the SQL backend for the Identity module. + +Likewise, ``test_cli.py`` takes advantage of the tests written aainst ``test_keystoneclient`` to verify the same tests function through different drivers. Testing Schema Migrations ------------------------- @@ -92,8 +100,9 @@ This command references to a SQLite database (test.db) to be used. Depending on Writing Tests ------------- -UPDATE THIS... +To add tests covering all drivers, update the base test class (``test_backend.py``, ``test_legacy_compat.py``, and ``test_keystoneclient.py``). +To add new drivers, subclass the ``test_backend.py`` (look towards ``test_backend_sql.py`` or ``test_backend_kvs.py`` for examples) and update the configuration of the test class in ``setUp()``. Further Testing --------------- From 9d7d898586566cf2f7c21846961f67a773c2ee6c Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 30 Jan 2012 19:53:08 +0000 Subject: [PATCH 267/334] shifting contents from _static to static --- docs/source/conf.py | 2 +- docs/source/{_static => static}/basic.css | 0 docs/source/{_static => static}/default.css | 0 docs/source/{_static => static}/jquery.tweet.js | 0 docs/source/{_static => static}/tweaks.css | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename docs/source/{_static => static}/basic.css (100%) rename docs/source/{_static => static}/default.css (100%) rename docs/source/{_static => static}/jquery.tweet.js (100%) rename docs/source/{_static => static}/tweaks.css (100%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6351572659..edae0b8a16 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -144,7 +144,7 @@ html_theme = '_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static', 'images'] +html_static_path = ['static', 'images'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/source/_static/basic.css b/docs/source/static/basic.css similarity index 100% rename from docs/source/_static/basic.css rename to docs/source/static/basic.css diff --git a/docs/source/_static/default.css b/docs/source/static/default.css similarity index 100% rename from docs/source/_static/default.css rename to docs/source/static/default.css diff --git a/docs/source/_static/jquery.tweet.js b/docs/source/static/jquery.tweet.js similarity index 100% rename from docs/source/_static/jquery.tweet.js rename to docs/source/static/jquery.tweet.js diff --git a/docs/source/_static/tweaks.css b/docs/source/static/tweaks.css similarity index 100% rename from docs/source/_static/tweaks.css rename to docs/source/static/tweaks.css From d961f7c30c1290dba392381a56c7159a09d36068 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 30 Jan 2012 19:59:58 +0000 Subject: [PATCH 268/334] pep8 cleanup --- keystone/middleware/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 305a7d4446..73dcc00344 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -103,8 +103,8 @@ class JsonBodyMiddleware(wsgi.Middleware): class Debug(wsgi.Middleware): """ - Middleware that produces stream debugging traces to the console (stdout) - for HTTP requests and responses flowing through it. + Middleware that produces stream debugging traces to the console (stdout) + for HTTP requests and responses flowing through it. """ @webob.dec.wsgify From 8d695b83df68367be9172f0659a90a844680039b Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 30 Jan 2012 20:22:34 +0000 Subject: [PATCH 269/334] removing unused images, cleaning up RST in docstrings from sphinx warnings --- docs/source/configuration.rst | 36 +-- docs/source/images/305.svg | 158 ------------ docs/source/images/both.svg | 135 ---------- docs/source/images/graphs_delegate_accept.svg | 52 ---- docs/source/images/graphs_separate.svg | 30 --- docs/source/images/graphs_standard_accept.svg | 51 ---- docs/source/images/graphs_standard_reject.svg | 39 --- docs/source/images/graphs_together.svg | 24 -- docs/source/images/layouts.svg | 215 ---------------- docs/source/images/mapper.svg | 237 ----------------- docs/source/images/proxyAuth.svg | 238 ------------------ keystone/catalog/backends/templated.py | 3 + keystone/common/cfg.py | 24 +- keystone/service.py | 18 +- 14 files changed, 45 insertions(+), 1215 deletions(-) delete mode 100644 docs/source/images/305.svg delete mode 100644 docs/source/images/both.svg delete mode 100644 docs/source/images/graphs_delegate_accept.svg delete mode 100644 docs/source/images/graphs_separate.svg delete mode 100644 docs/source/images/graphs_standard_accept.svg delete mode 100644 docs/source/images/graphs_standard_reject.svg delete mode 100644 docs/source/images/graphs_together.svg delete mode 100644 docs/source/images/layouts.svg delete mode 100644 docs/source/images/mapper.svg delete mode 100644 docs/source/images/proxyAuth.svg diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 410e7b1bfe..782e9fb572 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -31,7 +31,7 @@ initializing data into keystone using the command line client. Keystone Configuration File =========================== -The keystone configuration file is an 'ini' file format with sections, +The keystone configuration file is an 'ini' file format with sections, extended from Paste_, a common system used to configure python WSGI based applications. In addition to the paste config entries, general configuration values are stored under [DEFAULT] and [sql], and then drivers for the various @@ -44,7 +44,7 @@ The driver sections include: * ``[policy]`` - the python module that drives the policy system for RBAC * ``[ec2]`` - the python module providing the EC2 translations for OpenStack -The keystone configuration file is expected to be named ``keystone.conf``. +The keystone configuration file is expected to be named ``keystone.conf``. When starting up Keystone, you can specify a different configuration file to use with ``--config-file``. If you do **not** specify a configuration file, keystone will look in the following directories for a configuration file, in @@ -57,7 +57,7 @@ order: Logging is configured externally to the rest of keystone, the file specifying the logging configuration is in the [DEFAULT] section of the keystone conf -file under ``log_config``. If you wish to route all your logging through +file under ``log_config``. If you wish to route all your logging through syslog, there is a ``use_syslog`` option also in the [DEFAULT] section that easy. @@ -81,20 +81,26 @@ Sample Configuration Files Initializing Keystone ===================== -Keystone must be running in order to initialize data within it. This is because -the keystone-manage commands are all used the same REST API that other +Keystone must be running in order to initialize data within it. This is +because the keystone-manage commands are all used the same REST API that other OpenStack systems utilize. General keystone-manage options: -------------------------------- * ``--id-only`` : causes ``keystone-manage`` to return only the UUID result -from the API call. -* ``--endpoint`` : allows you to specify the keystone endpoint to communicate with. The default endpoint is http://localhost:35357/v2.0' + from the API call. + +* ``--endpoint`` : allows you to specify the keystone endpoint to communicate + with. The default endpoint is http://localhost:35357/v2.0' + * ``--auth-token`` : provides the authorization token -``keystone-manage`` is set up to expect commands in the general form of ``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to provide additional information to the command. For example, the command -``tenant`` has the subcommand ``create``, which takes the required keyword ``tenant_name``:: +``keystone-manage`` is set up to expect commands in the general form of +``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to +provide additional information to the command. For example, the command +``tenant`` has the subcommand ``create``, which takes the required keyword +``tenant_name``:: keystone-manage tenant create tenant_name=example_tenant @@ -137,7 +143,7 @@ creates a tenant named "admin". keyword arguments * tenant_id - + example:: keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 @@ -169,14 +175,14 @@ keyword arguments * name * password * email - + example:: keystone-manage user --ks-id-only create \ name=admin \ password=secrete \ email=admin@example.com - + ``user delete`` ^^^^^^^^^^^^^^^ @@ -199,7 +205,7 @@ keyword arguments ``user update_password`` ^^^^^^^^^^^^^^^^^^^^^^^^ - + keyword arguments ``user update_tenant`` @@ -220,7 +226,7 @@ keyword arguments exmaple:: keystone-manage role --ks-id-only create name=Admin - + ``role add_user_to_tenant`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -236,7 +242,7 @@ example:: role_id=19d1d3344873464d819c45f521ff9890 \ user_id=08741d8ed88242ca88d1f61484a0fe3b \ tenant_id=20601a7f1d94447daa4dff438cb1c209 - + ``role remove_user_from_tenant`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/images/305.svg b/docs/source/images/305.svg deleted file mode 100644 index 7d79464e2b..0000000000 --- a/docs/source/images/305.svg +++ /dev/null @@ -1,158 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - Request - service directly - - - Auth - Component - 305 - Use proxy to - redirect to Auth - - - - - - - OpenStack - Service - - - diff --git a/docs/source/images/both.svg b/docs/source/images/both.svg deleted file mode 100644 index d29872a4a6..0000000000 --- a/docs/source/images/both.svg +++ /dev/null @@ -1,135 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - Auth - Component - - Auth - Component - - - OpenStack - Service - - - - - diff --git a/docs/source/images/graphs_delegate_accept.svg b/docs/source/images/graphs_delegate_accept.svg deleted file mode 100644 index 1d86cadfc6..0000000000 --- a/docs/source/images/graphs_delegate_accept.svg +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - -DelegateAcceptAuth - - - -AuthComp - -Auth -Component - - -Start->AuthComp - - -Authorization: Basic VTpQ - - -AuthComp->Start - - -200 Okay - - -Service - -OpenStack -Service - - -AuthComp->Service - - -Authorization: Basic dTpw -X-Authorization: Proxy U -X-Identity-Status: Confirmed - - -Service->AuthComp - - -200 Okay - - - diff --git a/docs/source/images/graphs_separate.svg b/docs/source/images/graphs_separate.svg deleted file mode 100644 index 376e59880a..0000000000 --- a/docs/source/images/graphs_separate.svg +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - -Seperate - - -AuthComp - -Auth -Component - - -Service - -OpenStack -Service - - -AuthComp->Service - - - - - diff --git a/docs/source/images/graphs_standard_accept.svg b/docs/source/images/graphs_standard_accept.svg deleted file mode 100644 index bddf4b5f16..0000000000 --- a/docs/source/images/graphs_standard_accept.svg +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - -StandardAcceptAuth - - - -AuthComp - -Auth -Component - - -Start->AuthComp - - -Authorization: Basic VTpQ - - -AuthComp->Start - - -200 Okay - - -Service - -OpenStack -Service - - -AuthComp->Service - - -Authorization: Basic dTpw -X-Authorization: Proxy U - - -Service->AuthComp - - -200 Okay - - - diff --git a/docs/source/images/graphs_standard_reject.svg b/docs/source/images/graphs_standard_reject.svg deleted file mode 100644 index 6020ad67a5..0000000000 --- a/docs/source/images/graphs_standard_reject.svg +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - -StandardRejectAuth - - - -AuthComp - -Auth -Component - - -Start->AuthComp - - -Authorization: Basic Yjpw - - -AuthComp->Start - - -401 Unauthorized -WWW-Authenticate: Basic Realm="API Realm" - - -Service - -OpenStack -Service - - - diff --git a/docs/source/images/graphs_together.svg b/docs/source/images/graphs_together.svg deleted file mode 100644 index 1425a28baa..0000000000 --- a/docs/source/images/graphs_together.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - -Together - - -Together - - -Auth -Component - - -OpenStack -Service - - - diff --git a/docs/source/images/layouts.svg b/docs/source/images/layouts.svg deleted file mode 100644 index fdf61b7da7..0000000000 --- a/docs/source/images/layouts.svg +++ /dev/null @@ -1,215 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - Auth - Component - - - OpenStack - Service - - - Option - ( - b - ) - - - Auth - Component - - - OpenStack - Service - Option - ( - a - ) - - - - diff --git a/docs/source/images/mapper.svg b/docs/source/images/mapper.svg deleted file mode 100644 index b5a2b7b12f..0000000000 --- a/docs/source/images/mapper.svg +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - OpenStack - Service - - - - - - - - - - - - - - - Mapper - - - Auth - 1 - - - Auth - 2 - - - Auth - 3 - - - - - diff --git a/docs/source/images/proxyAuth.svg b/docs/source/images/proxyAuth.svg deleted file mode 100644 index f60b40d813..0000000000 --- a/docs/source/images/proxyAuth.svg +++ /dev/null @@ -1,238 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - Authorization - : - Basic dTpw - X - - - Authorization - : - Proxy U - Authorization - : - Basic VTpQ - 500 - Internal Error - 403 - Proxy Unauthorized - - - - - Auth - Component - - - - - OpenStack - Service - - - - - - - diff --git a/keystone/catalog/backends/templated.py b/keystone/catalog/backends/templated.py index 92c52574a2..e8290212ae 100644 --- a/keystone/catalog/backends/templated.py +++ b/keystone/catalog/backends/templated.py @@ -32,8 +32,11 @@ class TemplatedCatalog(kvs.Catalog): name - the name of the service, most likely repeated for all services of the same type, across regions. + adminURL - the url of the admin endpoint + publicURL - the url of the public endpoint + internalURL - the url of the internal endpoint """ diff --git a/keystone/common/cfg.py b/keystone/common/cfg.py index fd79e1222d..9551ff7967 100644 --- a/keystone/common/cfg.py +++ b/keystone/common/cfg.py @@ -17,7 +17,7 @@ r""" Configuration options which may be set on the command line or in config files. -The schema for each option is defined using the Opt sub-classes e.g. +The schema for each option is defined using the Opt sub-classes e.g.:: common_opts = [ cfg.StrOpt('bind_host', @@ -28,7 +28,7 @@ The schema for each option is defined using the Opt sub-classes e.g. help='Port number to listen on') ] -Options can be strings, integers, floats, booleans, lists or 'multi strings': +Options can be strings, integers, floats, booleans, lists or 'multi strings':: enabled_apis_opt = \ cfg.ListOpt('enabled_apis', @@ -43,7 +43,7 @@ Options can be strings, integers, floats, booleans, lists or 'multi strings': default=DEFAULT_EXTENSIONS) Option schemas are registered with with the config manager at runtime, but -before the option is referenced: +before the option is referenced:: class ExtensionManager(object): @@ -59,7 +59,7 @@ before the option is referenced: .... A common usage pattern is for each option schema to be defined in the module or -class which uses the option: +class which uses the option:: opts = ... @@ -74,7 +74,7 @@ class which uses the option: An option may optionally be made available via the command line. Such options must registered with the config manager before the command line is parsed (for -the purposes of --help and CLI arg validation): +the purposes of --help and CLI arg validation):: cli_opts = [ cfg.BoolOpt('verbose', @@ -90,7 +90,7 @@ the purposes of --help and CLI arg validation): def add_common_opts(conf): conf.register_cli_opts(cli_opts) -The config manager has a single CLI option defined by default, --config-file: +The config manager has a single CLI option defined by default, --config-file:: class ConfigOpts(object): @@ -104,7 +104,7 @@ The config manager has a single CLI option defined by default, --config-file: Option values are parsed from any supplied config files using SafeConfigParser. If none are specified, a default set is used e.g. glance-api.conf and -glance-common.conf: +glance-common.conf:: glance-api.conf: [DEFAULT] @@ -119,7 +119,7 @@ are parsed in order, with values in later files overriding those in earlier files. The parsing of CLI args and config files is initiated by invoking the config -manager e.g. +manager e.g.:: conf = ConfigOpts() conf.register_opt(BoolOpt('verbose', ...)) @@ -127,7 +127,7 @@ manager e.g. if conf.verbose: ... -Options can be registered as belonging to a group: +Options can be registered as belonging to a group:: rabbit_group = cfg.OptionGroup(name='rabbit', title='RabbitMQ options') @@ -154,7 +154,7 @@ Options can be registered as belonging to a group: conf.register_opt(rabbit_ssl_opt, group=rabbit_group) If no group is specified, options belong to the 'DEFAULT' section of config -files: +files:: glance-api.conf: [DEFAULT] @@ -175,7 +175,7 @@ Command-line options in a group are automatically prefixed with the group name: Option values in the default group are referenced as attributes/properties on the config manager; groups are also attributes on the config manager, with -attributes for each of the options associated with the group: +attributes for each of the options associated with the group:: server.start(app, conf.bind_port, conf.bind_host, conf) @@ -184,7 +184,7 @@ attributes for each of the options associated with the group: port=conf.rabbit.port, ...) -Option values may reference other values using PEP 292 string substitution: +Option values may reference other values using PEP 292 string substitution:: opts = [ cfg.StrOpt('state_path', diff --git a/keystone/service.py b/keystone/service.py index b017d69768..05cca4e600 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -109,17 +109,17 @@ class TokenController(wsgi.Application): def authenticate(self, context, auth=None): """Authenticate credentials and return a token. - Accept auth as a dict that looks like: + Accept auth as a dict that looks like:: - { - "auth":{ - "passwordCredentials":{ - "username":"test_user", - "password":"mypass" - }, - "tenantName":"customer-x" + { + "auth":{ + "passwordCredentials":{ + "username":"test_user", + "password":"mypass" + }, + "tenantName":"customer-x" + } } - } In this case, tenant is optional, if not provided the token will be considered "unscoped" and can later be used to get a scoped token. From 666a2b8d8f275bdd80ab12e4686b0892673e841c Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 31 Jan 2012 02:12:38 -0800 Subject: [PATCH 270/334] Add .gitreview file. Change-Id: I022e6b0dfe3db6dc5c1530cfbbf3f7d5e6804962 --- .gitreview | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitreview diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000000..cd914fe02a --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/keystone.git From cf3f671a050f5cb5a2acc8c8c4b0b6b7a3a0d892 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 31 Jan 2012 13:39:36 -0500 Subject: [PATCH 271/334] Fix pep8 violations. Change-Id: I12e304c567b92178e193c60599c3be606cc70d38 --- bin/keystone-manage | 14 +- keystone/cli.py | 463 +++++++++++++++++++------------------ keystone/common/logging.py | 16 +- keystone/common/utils.py | 54 ++--- setup.py | 3 +- 5 files changed, 276 insertions(+), 274 deletions(-) diff --git a/bin/keystone-manage b/bin/keystone-manage index bcc27195eb..e551c47e2f 100755 --- a/bin/keystone-manage +++ b/bin/keystone-manage @@ -18,11 +18,11 @@ from keystone import cli if __name__ == '__main__': - dev_conf = os.path.join(possible_topdir, - 'etc', - 'keystone.conf') - config_files = None - if os.path.exists(dev_conf): - config_files = [dev_conf] + dev_conf = os.path.join(possible_topdir, + 'etc', + 'keystone.conf') + config_files = None + if os.path.exists(dev_conf): + config_files = [dev_conf] - cli.main(argv=sys.argv, config_files=config_files) + cli.main(argv=sys.argv, config_files=config_files) diff --git a/keystone/cli.py b/keystone/cli.py index 93e3620564..e178a31b64 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -33,316 +33,317 @@ config.register_cli_bool('id-only', class BaseApp(cli.log.LoggingApp): - def __init__(self, *args, **kw): - kw.setdefault('name', self.__class__.__name__.lower()) - super(BaseApp, self).__init__(*args, **kw) + def __init__(self, *args, **kw): + kw.setdefault('name', self.__class__.__name__.lower()) + super(BaseApp, self).__init__(*args, **kw) - def add_default_params(self): - for args, kw in DEFAULT_PARAMS: - self.add_param(*args, **kw) + def add_default_params(self): + for args, kw in DEFAULT_PARAMS: + self.add_param(*args, **kw) - def _parse_keyvalues(self, args): - kv = {} - for x in args: - key, value = x.split('=', 1) - # make lists if there are multiple values - if key.endswith('[]'): - key = key[:-2] - existing = kv.get(key, []) - existing.append(value) - kv[key] = existing - else: - kv[key] = value - return kv + def _parse_keyvalues(self, args): + kv = {} + for x in args: + key, value = x.split('=', 1) + # make lists if there are multiple values + if key.endswith('[]'): + key = key[:-2] + existing = kv.get(key, []) + existing.append(value) + kv[key] = existing + else: + kv[key] = value + return kv class DbSync(BaseApp): - """Sync the database.""" + """Sync the database.""" - name = 'db_sync' + name = 'db_sync' - def __init__(self, *args, **kw): - super(DbSync, self).__init__(*args, **kw) + def __init__(self, *args, **kw): + super(DbSync, self).__init__(*args, **kw) - def main(self): - for k in ['identity', 'catalog', 'policy', 'token']: - driver = utils.import_object(getattr(CONF, k).driver) - if hasattr(driver, 'db_sync'): - driver.db_sync() + def main(self): + for k in ['identity', 'catalog', 'policy', 'token']: + driver = utils.import_object(getattr(CONF, k).driver) + if hasattr(driver, 'db_sync'): + driver.db_sync() class ClientCommand(BaseApp): - ACTION_MAP = None + ACTION_MAP = None - def _attr_name(self): - return '%ss' % self.__class__.__name__.lower() + def _attr_name(self): + return '%ss' % self.__class__.__name__.lower() - def _cmd_name(self): - return self.__class__.__name__.lower() + def _cmd_name(self): + return self.__class__.__name__.lower() - def __init__(self, *args, **kw): - super(ClientCommand, self).__init__(*args, **kw) - if not self.ACTION_MAP: - self.ACTION_MAP = {'help': 'help'} - self.add_param('action', nargs='?', default='help') - self.add_param('keyvalues', nargs='*') - self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) - self.handle = getattr(self.client, self._attr_name()) - self._build_action_map() + def __init__(self, *args, **kw): + super(ClientCommand, self).__init__(*args, **kw) + if not self.ACTION_MAP: + self.ACTION_MAP = {'help': 'help'} + self.add_param('action', nargs='?', default='help') + self.add_param('keyvalues', nargs='*') + self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) + self.handle = getattr(self.client, self._attr_name()) + self._build_action_map() - def _build_action_map(self): - actions = {} - for k in dir(self.handle): - if not k.startswith('_'): - actions[k] = k - self.ACTION_MAP.update(actions) + def _build_action_map(self): + actions = {} + for k in dir(self.handle): + if not k.startswith('_'): + actions[k] = k + self.ACTION_MAP.update(actions) - def main(self): - """Given some keyvalues create the appropriate data in Keystone.""" - action_name = self.ACTION_MAP[self.params.action] - if action_name == 'help': - self.print_help() - sys.exit(1) + def main(self): + """Given some keyvalues create the appropriate data in Keystone.""" + action_name = self.ACTION_MAP[self.params.action] + if action_name == 'help': + self.print_help() + sys.exit(1) - kv = self._parse_keyvalues(self.params.keyvalues) - try: - f = getattr(self.handle, action_name) - resp = f(**kv) - except Exception: - logging.exception('') - raise + kv = self._parse_keyvalues(self.params.keyvalues) + try: + f = getattr(self.handle, action_name) + resp = f(**kv) + except Exception: + logging.exception('') + raise - if CONF.id_only and getattr(resp, 'id'): - print resp.id - return + if CONF.id_only and getattr(resp, 'id'): + print resp.id + return - if resp is None: - return + if resp is None: + return - # NOTE(termie): this is ugly but it is mostly because the keystoneclient - # code doesn't give us very serializable instance objects - if type(resp) in [type(list()), type(tuple())]: - o = [] - for r in resp: - d = {} - for k, v in sorted(r.__dict__.iteritems()): - if k[0] == '_' or k == 'manager': - continue - d[k] = v - o.append(d) - else: - o = {} - for k, v in sorted(resp.__dict__.iteritems()): - if k[0] == '_' or k == 'manager': - continue - o[k] = v + # NOTE(termie): this is ugly but it is mostly because the + # keystoneclient code doesn't give us very + # serializable instance objects + if type(resp) in [type(list()), type(tuple())]: + o = [] + for r in resp: + d = {} + for k, v in sorted(r.__dict__.iteritems()): + if k[0] == '_' or k == 'manager': + continue + d[k] = v + o.append(d) + else: + o = {} + for k, v in sorted(resp.__dict__.iteritems()): + if k[0] == '_' or k == 'manager': + continue + o[k] = v - print json.dumps(o) + print json.dumps(o) - def print_help(self): - CONF.set_usage(CONF.usage.replace( - 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) - CONF.print_help() + def print_help(self): + CONF.set_usage(CONF.usage.replace( + 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) + CONF.print_help() - methods = self._get_methods() - print_commands(methods) + methods = self._get_methods() + print_commands(methods) - def _get_methods(self): - o = {} - for k in dir(self.handle): - if k.startswith('_'): - continue - if k in ('find', 'findall', 'api', 'resource_class'): - continue - o[k] = getattr(self.handle, k) - return o + def _get_methods(self): + o = {} + for k in dir(self.handle): + if k.startswith('_'): + continue + if k in ('find', 'findall', 'api', 'resource_class'): + continue + o[k] = getattr(self.handle, k) + return o class Role(ClientCommand): - """Role CRUD functions.""" - pass + """Role CRUD functions.""" + pass class Service(ClientCommand): - """Service CRUD functions.""" - pass + """Service CRUD functions.""" + pass class Token(ClientCommand): - """Token CRUD functions.""" - pass + """Token CRUD functions.""" + pass class Tenant(ClientCommand): - """Tenant CRUD functions.""" - pass + """Tenant CRUD functions.""" + pass class User(ClientCommand): - """User CRUD functions.""" + """User CRUD functions.""" - pass + pass class Ec2(ClientCommand): - def _attr_name(self): - return self.__class__.__name__.lower() + def _attr_name(self): + return self.__class__.__name__.lower() CMDS = {'db_sync': DbSync, - 'role': Role, - 'service': Service, - 'token': Token, - 'tenant': Tenant, - 'user': User, - 'ec2': Ec2, - } + 'role': Role, + 'service': Service, + 'token': Token, + 'tenant': Tenant, + 'user': User, + 'ec2': Ec2, + } class CommandLineGenerator(object): - """A keystoneclient lookalike to generate keystone-manage commands. + """A keystoneclient lookalike to generate keystone-manage commands. - One would use it like so: + One would use it like so: - >>> gen = CommandLineGenerator(id_only=None) - >>> cl = gen.ec2.create(user_id='foo', tenant_id='foo') - >>> cl.to_argv() - ... ['keystone-manage', - '--id-only', - 'ec2', - 'create', - 'user_id=foo', - 'tenant_id=foo'] + >>> gen = CommandLineGenerator(id_only=None) + >>> cl = gen.ec2.create(user_id='foo', tenant_id='foo') + >>> cl.to_argv() + ... ['keystone-manage', + '--id-only', + 'ec2', + 'create', + 'user_id=foo', + 'tenant_id=foo'] - """ + """ - cmd = 'keystone-manage' + cmd = 'keystone-manage' - def __init__(self, cmd=None, execute=False, **kw): - if cmd: - self.cmd = cmd - self.flags = kw - self.execute = execute + def __init__(self, cmd=None, execute=False, **kw): + if cmd: + self.cmd = cmd + self.flags = kw + self.execute = execute - def __getattr__(self, key): - return _Manager(self, key) + def __getattr__(self, key): + return _Manager(self, key) class _Manager(object): - def __init__(self, parent, name): - self.parent = parent - self.name = name + def __init__(self, parent, name): + self.parent = parent + self.name = name - def __getattr__(self, key): - return _CommandLine(cmd=self.parent.cmd, - flags=self.parent.flags, - manager=self.name, - method=key, - execute=self.parent.execute) + def __getattr__(self, key): + return _CommandLine(cmd=self.parent.cmd, + flags=self.parent.flags, + manager=self.name, + method=key, + execute=self.parent.execute) class _CommandLine(object): - def __init__(self, cmd, flags, manager, method, execute=False): - self.cmd = cmd - self.flags = flags - self.manager = manager - self.method = method - self.execute = execute - self.kw = {} + def __init__(self, cmd, flags, manager, method, execute=False): + self.cmd = cmd + self.flags = flags + self.manager = manager + self.method = method + self.execute = execute + self.kw = {} - def __call__(self, **kw): - self.kw = kw - if self.execute: - logging.debug('generated cli: %s', str(self)) - out = StringIO.StringIO() - old_out = sys.stdout - sys.stdout = out - try: - main(self.to_argv()) - except SystemExit as e: - pass - finally: - sys.stdout = old_out - rv = out.getvalue().strip().split('\n')[-1] - try: - loaded = json.loads(rv) - if type(loaded) in [type(list()), type(tuple())]: - return [DictWrapper(**x) for x in loaded] - elif type(loaded) is type(dict()): - return DictWrapper(**loaded) - except Exception: - logging.exception('Could not parse JSON: %s', rv) - return rv - return self + def __call__(self, **kw): + self.kw = kw + if self.execute: + logging.debug('generated cli: %s', str(self)) + out = StringIO.StringIO() + old_out = sys.stdout + sys.stdout = out + try: + main(self.to_argv()) + except SystemExit as e: + pass + finally: + sys.stdout = old_out + rv = out.getvalue().strip().split('\n')[-1] + try: + loaded = json.loads(rv) + if type(loaded) in [type(list()), type(tuple())]: + return [DictWrapper(**x) for x in loaded] + elif type(loaded) is type(dict()): + return DictWrapper(**loaded) + except Exception: + logging.exception('Could not parse JSON: %s', rv) + return rv + return self - def __flags(self): - o = [] - for k, v in self.flags.iteritems(): - k = k.replace('_', '-') - if v is None: - o.append('--%s' % k) - else: - o.append('--%s=%s' % (k, str(v))) - return o + def __flags(self): + o = [] + for k, v in self.flags.iteritems(): + k = k.replace('_', '-') + if v is None: + o.append('--%s' % k) + else: + o.append('--%s=%s' % (k, str(v))) + return o - def __manager(self): - if self.manager.endswith('s'): - return self.manager[:-1] - return self.manager + def __manager(self): + if self.manager.endswith('s'): + return self.manager[:-1] + return self.manager - def __kw(self): - o = [] - for k, v in self.kw.iteritems(): - o.append('%s=%s' % (k, str(v))) - return o + def __kw(self): + o = [] + for k, v in self.kw.iteritems(): + o.append('%s=%s' % (k, str(v))) + return o - def to_argv(self): - return ([self.cmd] - + self.__flags() - + [self.__manager(), self.method] - + self.__kw()) + def to_argv(self): + return ([self.cmd] + + self.__flags() + + [self.__manager(), self.method] + + self.__kw()) - def __str__(self): - args = self.to_argv() - return ' '.join(args[:1] + ['"%s"' % x for x in args[1:]]) + def __str__(self): + args = self.to_argv() + return ' '.join(args[:1] + ['"%s"' % x for x in args[1:]]) class DictWrapper(dict): - def __getattr__(self, key): - try: - return self[key] - except KeyError: - raise AttributeError(key) + def __getattr__(self, key): + try: + return self[key] + except KeyError: + raise AttributeError(key) def print_commands(cmds): - print - print "Available commands:" - o = [] - max_length = max([len(k) for k in cmds]) + 2 - for k, cmd in sorted(cmds.iteritems()): - initial_indent = '%s%s: ' % (' ' * (max_length - len(k)), k) - tw = textwrap.TextWrapper(initial_indent=initial_indent, - subsequent_indent=' ' * (max_length + 2), - width=80) - o.extend(tw.wrap( - (cmd.__doc__ and cmd.__doc__ or 'no docs').strip().split('\n')[0])) - print '\n'.join(o) + print + print "Available commands:" + o = [] + max_length = max([len(k) for k in cmds]) + 2 + for k, cmd in sorted(cmds.iteritems()): + initial_indent = '%s%s: ' % (' ' * (max_length - len(k)), k) + tw = textwrap.TextWrapper(initial_indent=initial_indent, + subsequent_indent=' ' * (max_length + 2), + width=80) + o.extend(tw.wrap( + (cmd.__doc__ and cmd.__doc__ or 'no docs').strip().split('\n')[0])) + print '\n'.join(o) def run(cmd, args): - return CMDS[cmd](argv=args).run() + return CMDS[cmd](argv=args).run() def main(argv=None, config_files=None): - CONF.reset() - args = CONF(config_files=config_files, args=argv) + CONF.reset() + args = CONF(config_files=config_files, args=argv) - if len(args) < 2: - CONF.print_help() - print_commands(CMDS) - sys.exit(1) + if len(args) < 2: + CONF.print_help() + print_commands(CMDS) + sys.exit(1) - cmd = args[1] - if cmd in CMDS: - return run(cmd, (args[:1] + args[2:])) + cmd = args[1] + if cmd in CMDS: + return run(cmd, (args[:1] + args[2:])) diff --git a/keystone/common/logging.py b/keystone/common/logging.py index 7ef073763b..a9aaccfced 100644 --- a/keystone/common/logging.py +++ b/keystone/common/logging.py @@ -45,11 +45,11 @@ SysLogHandler = SysLogHandler def log_debug(f): - @functools.wraps(f) - def wrapper(*args, **kw): - logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw)) - rv = f(*args, **kw) - logging.debug(pprint.pformat(rv, indent=2)) - logging.debug('') - return rv - return wrapper + @functools.wraps(f) + def wrapper(*args, **kw): + logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw)) + rv = f(*args, **kw) + logging.debug(pprint.pformat(rv, indent=2)) + logging.debug('') + return rv + return wrapper diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 3ee9489441..95a1b97103 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -161,38 +161,38 @@ def check_password(password, hashed): # From python 2.7 def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. + r"""Run command with arguments and return its output as a byte string. - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. - The arguments are the same as for the Popen constructor. Example: + The arguments are the same as for the Popen constructor. Example: - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - logging.debug(' '.join(popenargs[0])) - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + logging.debug(' '.join(popenargs[0])) + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output def git(*args): - return check_output(['git'] + list(args)) + return check_output(['git'] + list(args)) diff --git a/setup.py b/setup.py index 6c1ce8486d..5f52c70252 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ from setuptools import setup, find_packages + setup(name='keystone', version='2012.1', description="Authentication service for OpenStack", @@ -10,5 +11,5 @@ setup(name='keystone', packages=find_packages(exclude=['test', 'bin']), scripts=['bin/keystone', 'bin/keystone-manage'], zip_safe=False, - install_requires = ['setuptools', 'python-keystoneclient'], + install_requires=['setuptools', 'python-keystoneclient'], ) From fc3de2491d15f446d6223ff494bbeaef06fda8ac Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 31 Jan 2012 13:40:29 -0500 Subject: [PATCH 272/334] Use gerrit instead of github When we run gating tests in jenkins, any access of network resources is a potential source of false-negative on the test due to intermittent service failures on systems that are out of our control. We observe that this is actually quite frequent when things want to access PyPI or github. With pypi, we pre-create virtualenvs and cache the eggs so that an individual test run doesn't fail due to pypi not responding. For repos, if at all possible, we direct them all at the gerrit instance, because since gerrit is driving the test run in the first place, it's indicative of a much larger problem if jenkins can't talk to it - and it's one that we can fix if it does come up. Change-Id: I9f54133f7f2025d15a9d0b270d2466438cbc6dd5 --- tests/test_keystoneclient.py | 3 ++- tests/test_legacy_compat.py | 8 ++++---- tests/test_novaclient_compat.py | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 9766108ad4..d7748649d9 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -7,7 +7,8 @@ from keystone import test import default_fixtures CONF = config.CONF -KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' +OPENSTACK_REPO = 'https://review.openstack.org/p/openstack' +KEYSTONECLIENT_REPO = '%s/python-keystoneclient.git' % OPENSTACK_REPO class CompatTestCase(test.TestCase): diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py index ff6b7fdbc2..e9d9ebc0f3 100644 --- a/tests/test_legacy_compat.py +++ b/tests/test_legacy_compat.py @@ -13,11 +13,11 @@ from keystone.common import utils CONF = config.CONF +OPENSTACK_REPO = 'https://review.openstack.org/p/openstack/' -IDENTITY_API_REPO = 'git://github.com/openstack/identity-api.git' -KEYSTONE_REPO = 'git://github.com/openstack/keystone.git' -NOVACLIENT_REPO = 'git://github.com/rackspace/python-novaclient.git' - +IDENTITY_API_REPO = '%s/identity-api.git' % OPENSTACK_REPO +KEYSTONE_REPO = '%s/keystone.git' % OPENSTACK_REPO +NOVACLIENT_REPO = '%s/python-novaclient.git' % OPENSTACK_REPO IDENTITY_SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' KEYSTONE_SAMPLE_DIR = 'keystone/content/common/samples' diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py index b4f6dc7928..5578e4f784 100644 --- a/tests/test_novaclient_compat.py +++ b/tests/test_novaclient_compat.py @@ -12,7 +12,8 @@ import default_fixtures CONF = config.CONF -NOVACLIENT_REPO = 'git://github.com/openstack/python-novaclient.git' +OPENSTACK_REPO = 'https://review.openstack.org/p/openstack' +NOVACLIENT_REPO = '%s/python-novaclient.git' % OPENSTACK_REPO class CompatTestCase(test.TestCase): From 3da657555d801a487fbfbc97778e2664e50f8434 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 31 Jan 2012 13:45:53 -0500 Subject: [PATCH 273/334] Normalize build files with current jenkins. Change-Id: I528c3cc4e16dfa1465c8e3ac1062c65dc2ddc2f0 --- .gitignore | 3 +++ run_tests.sh | 3 +-- tools/install_venv.py | 4 ++-- tools/pip-requires | 23 ++++++++++++++++++++++- tools/with_venv.sh | 2 +- tox.ini | 26 ++++++++++++++++++++++++++ 6 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 6dc19a6eb0..cd78247219 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ *.swp vendor .ksl-venv +.venv +.tox +keystone.egg-info/ run_tests.log .coverage covhtml diff --git a/run_tests.sh b/run_tests.sh index 825d9501b1..c55a0cd9e5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -40,7 +40,7 @@ function process_option { esac } -venv=.ksl-venv +venv=.venv with_venv=tools/with_venv.sh always_venv=0 never_venv=0 @@ -89,7 +89,6 @@ function run_pep8 { srcfiles+=" keystone" # Just run PEP8 in current environment ${wrapper} pep8 --repeat --show-pep8 --show-source \ - --ignore=E202,E111 \ --exclude=vcsversion.py ${srcfiles} | tee pep8.txt } diff --git a/tools/install_venv.py b/tools/install_venv.py index c2489f29e1..11f682b63d 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -28,8 +28,8 @@ import sys ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.ksl-venv') -PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires-test') +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) diff --git a/tools/pip-requires b/tools/pip-requires index c075d369ee..94ab44ba2c 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,3 +1,4 @@ +# keystonelight dependencies pam==0.1.4 WebOb==0.9.8 eventlet==0.9.12 @@ -9,4 +10,24 @@ sqlalchemy sqlalchemy-migrate py-bcrypt --e git+https://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient +# for python-novaclient +prettytable + +# Optional backend: Memcache +python-memcached # increases performance of token validation calls + +# Development +Sphinx>=1.1.2 # required to build documentation +coverage # computes code coverage percentages + +# Testing +nose # for test discovery and console feedback +nosexcover +unittest2 # backport of unittest lib in python 2.7 +webtest # test wsgi apps without starting an http server +pylint # static code analysis +pep8==0.6.1 # checks for PEP8 code style compliance +mox # mock object framework + +-e git+https://review.openstack.org/p/openstack/python-keystoneclient.git#egg=python-keystoneclient +-e git+https://review.openstack.org/p/openstack-dev/openstack-nose.git#egg=openstack.nose_plugin diff --git a/tools/with_venv.sh b/tools/with_venv.sh index 0ed2ef7268..c8d2940fc7 100755 --- a/tools/with_venv.sh +++ b/tools/with_venv.sh @@ -1,4 +1,4 @@ #!/bin/bash TOOLS=`dirname $0` -VENV=$TOOLS/../.ksl-venv +VENV=$TOOLS/../.venv source $VENV/bin/activate && $@ diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..e9a2fb389b --- /dev/null +++ b/tox.ini @@ -0,0 +1,26 @@ +[tox] +envlist = py26,py27,pep8 + +[testenv] +deps = -r{toxinidir}/tools/pip-requires +commands = nosetests + +[testenv:pep8] +commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin keystone setup.py + +[testenv:hudson] +downloadcache = ~/cache/pip + +[testenv:jenkins26] +basepython = python2.6 +deps = file://{toxinidir}/.cache.bundle + +[testenv:jenkins27] +basepython = python2.7 +deps = file://{toxinidir}/.cache.bundle + +[testenv:jenkinspep8] +deps = file://{toxinidir}/.cache.bundle +commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin keystone setup.py + + From ec89d4ec9ae5428fe3509c5155d66efedb8a6668 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Wed, 1 Feb 2012 13:35:59 -0500 Subject: [PATCH 274/334] Change the name of keystone to keystone-server so the binaries dont conflict with python-keystoneclient. Signed-off-by: Chuck Short --- bin/{keystone => keystone-server} | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename bin/{keystone => keystone-server} (97%) diff --git a/bin/keystone b/bin/keystone-server similarity index 97% rename from bin/keystone rename to bin/keystone-server index 5f75279baf..d30d9c6192 100755 --- a/bin/keystone +++ b/bin/keystone-server @@ -12,7 +12,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) if os.path.exists(os.path.join(possible_topdir, - 'keystone', + 'keystone-server', '__init__.py')): sys.path.insert(0, possible_topdir) diff --git a/setup.py b/setup.py index 5f52c70252..fa5c3bea6d 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup(name='keystone', author_email='openstack@lists.launchpad.net', url='http://www.openstack.org', packages=find_packages(exclude=['test', 'bin']), - scripts=['bin/keystone', 'bin/keystone-manage'], + scripts=['bin/keystone-server', 'bin/keystone-manage'], zip_safe=False, install_requires=['setuptools', 'python-keystoneclient'], ) From 2d2ce8c72efa965b6db4affe1a5d9e3e6ca5d47b Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 1 Feb 2012 11:07:32 -0800 Subject: [PATCH 275/334] Update docs/source/developing.rst --- docs/source/developing.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/developing.rst b/docs/source/developing.rst index 344a6e8bc0..f529761eb0 100644 --- a/docs/source/developing.rst +++ b/docs/source/developing.rst @@ -54,6 +54,12 @@ You can also interact with Keystone through it's REST API. There is a python keystone client library `python-keystoneclient`_ which interacts exclusively through the REST API, and which keystone itself uses to provide it's command-line interface. +When initially getting set up, after you've configured which databases to use, +you're probably going to need to run the following to your database schema in place :: + + $ bin/keystone-manage db_sync + + .. _`python-keystoneclient`: https://github.com/openstack/python-keystoneclient Running Tests From c6e30eb5a1b14816e06589c407ea81f6d63b5355 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 31 Jan 2012 15:45:00 -0800 Subject: [PATCH 276/334] add tests for essex and fix the testing framework --- keystone/test.py | 5 +++++ tests/test_keystoneclient.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/keystone/test.py b/keystone/test.py index 4265514244..9f2773f7e8 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -197,6 +197,11 @@ class TestCase(unittest.TestCase): sys.path.insert(0, path) self._paths.append(path) + def clear_module(self, module): + for x in sys.modules.keys(): + if x.startswith(module): + del sys.modules[x] + def assertListEquals(self, actual, expected): copy = expected[:] #print expected, actual diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index d7748649d9..49873e1d93 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -41,10 +41,9 @@ class KcMasterTestCase(CompatTestCase): def setUp(self): super(KcMasterTestCase, self).setUp() - revdir = test.checkout_vendor(KEYSTONECLIENT_REPO, 'master') + revdir = test.checkout_vendor(*self.get_checkout()) self.add_path(revdir) - from keystoneclient.v2_0 import client as ks_client - reload(ks_client) + self.clear_module('keystoneclient') self.public_app = self.loadapp('keystone', name='main') self.admin_app = self.loadapp('keystone', name='admin') @@ -63,6 +62,9 @@ class KcMasterTestCase(CompatTestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) + def get_checkout(self): + return KEYSTONECLIENT_REPO, 'master' + def get_client(self, user_ref=None, tenant_ref=None): if user_ref is None: user_ref = self.user_foo @@ -429,3 +431,8 @@ class KcMasterTestCase(CompatTestCase): # TODO(ja): MEMBERSHIP CRUD # TODO(ja): determine what else todo + + +class KcEssex3TestCase(KcMasterTestCase): + def get_checkout(self): + return KEYSTONECLIENT_REPO, 'essex-3' From 6fd68e1a3878502c9d2bb65be670791e20a29ef6 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 31 Jan 2012 21:31:36 -0800 Subject: [PATCH 277/334] fix keystoneclient tests --- keystone/contrib/admin_crud/core.py | 2 +- keystone/identity/core.py | 57 ++++++++++- keystone/middleware/core.py | 2 + tests/test_keystoneclient.py | 148 ++++++++++++++++------------ 4 files changed, 145 insertions(+), 64 deletions(-) diff --git a/keystone/contrib/admin_crud/core.py b/keystone/contrib/admin_crud/core.py index f58b759e63..25be6b399f 100644 --- a/keystone/contrib/admin_crud/core.py +++ b/keystone/contrib/admin_crud/core.py @@ -110,7 +110,7 @@ class CrudExtension(wsgi.ExtensionRouter): conditions=dict(method=["PUT"])) mapper.connect( "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="delete_role_from_user", + controller=role_controller, action="remove_role_from_user", conditions=dict(method=["DELETE"])) # Service Operations diff --git a/keystone/identity/core.py b/keystone/identity/core.py index e841898545..ab37f93025 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -366,8 +366,20 @@ class RoleController(wsgi.Application): self.policy_api = policy.Manager() super(RoleController, self).__init__() + # COMPAT(essex-3) def get_user_roles(self, context, user_id, tenant_id=None): - raise NotImplemented() + """Get the roles for a user and tenant pair. + + Since we're trying to ignore the idea of user-only roles we're + not implementing them in hopes that the idea will die off. + + """ + if tenant_id is None: + raise Exception('User roles not supported: tenant_id required') + roles = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + return {'roles': [self.identity_api.get_role(context, x) + for x in roles]} # CRUD extension def get_role(self, context, role_id): @@ -395,6 +407,47 @@ class RoleController(wsgi.Application): # TODO(termie): probably inefficient at some point return {'roles': roles} + def add_role_to_user(self, context, user_id, role_id, tenant_id=None): + """Add a role to a user and tenant pair. + + Since we're trying to ignore the idea of user-only roles we're + not implementing them in hopes that the idea will die off. + + """ + self.assert_admin(context) + if tenant_id is None: + raise Exception('User roles not supported: tenant_id required') + + # This still has the weird legacy semantics that adding a role to + # a user also adds them to a tenant + self.identity_api.add_user_to_tenant(context, tenant_id, user_id) + self.identity_api.add_role_to_user_and_tenant( + context, user_id, tenant_id, role_id) + role_ref = self.identity_api.get_role(context, role_id) + return {'role': role_ref} + + def remove_role_from_user(self, context, user_id, role_id, tenant_id=None): + """Remove a role from a user and tenant pair. + + Since we're trying to ignore the idea of user-only roles we're + not implementing them in hopes that the idea will die off. + + """ + self.assert_admin(context) + if tenant_id is None: + raise Exception('User roles not supported: tenant_id required') + + # This still has the weird legacy semantics that adding a role to + # a user also adds them to a tenant + self.identity_api.remove_role_from_user_and_tenant( + context, user_id, tenant_id, role_id) + roles = self.identity_api.get_roles_for_user_and_tenant( + context, user_id, tenant_id) + if not roles: + self.identity_api.remove_user_from_tenant( + context, tenant_id, user_id) + return + # COMPAT(diablo): CRUD extension def get_role_refs(self, context, user_id): """Ultimate hack to get around having to make role_refs first-class. @@ -420,6 +473,7 @@ class RoleController(wsgi.Application): o.append(ref) return {'roles': o} + # COMPAT(diablo): CRUD extension def create_role_ref(self, context, user_id, role): """This is actually used for adding a user to a tenant. @@ -437,6 +491,7 @@ class RoleController(wsgi.Application): role_ref = self.identity_api.get_role(context, role_id) return {'role': role_ref} + # COMPAT(diablo): CRUD extension def delete_role_ref(self, context, user_id, role_ref_id): """This is actually used for deleting a user from a tenant. diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 73dcc00344..21e0f624c6 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -90,6 +90,8 @@ class JsonBodyMiddleware(wsgi.Middleware): return params_parsed = json.loads(params_json) + if not params_parsed: + params_parsed = {} params = {} for k, v in params_parsed.iteritems(): if k in ('self', 'context'): diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 49873e1d93..1cda1bc620 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -15,32 +15,6 @@ class CompatTestCase(test.TestCase): def setUp(self): super(CompatTestCase, self).setUp() - def _public_url(self): - public_port = self.public_server.socket_info['socket'][1] - CONF.public_port = public_port - return "http://localhost:%s/v2.0" % public_port - - def _admin_url(self): - admin_port = self.admin_server.socket_info['socket'][1] - CONF.admin_port = admin_port - return "http://localhost:%s/v2.0" % admin_port - - def _client(self, **kwargs): - from keystoneclient.v2_0 import client as ks_client - - kc = ks_client.Client(endpoint=self._admin_url(), - auth_url=self._public_url(), - **kwargs) - kc.authenticate() - # have to manually overwrite the management url after authentication - kc.management_url = self._admin_url() - return kc - - -class KcMasterTestCase(CompatTestCase): - def setUp(self): - super(KcMasterTestCase, self).setUp() - revdir = test.checkout_vendor(*self.get_checkout()) self.add_path(revdir) self.clear_module('keystoneclient') @@ -62,8 +36,26 @@ class KcMasterTestCase(CompatTestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) - def get_checkout(self): - return KEYSTONECLIENT_REPO, 'master' + def _public_url(self): + public_port = self.public_server.socket_info['socket'][1] + CONF.public_port = public_port + return "http://localhost:%s/v2.0" % public_port + + def _admin_url(self): + admin_port = self.admin_server.socket_info['socket'][1] + CONF.admin_port = admin_port + return "http://localhost:%s/v2.0" % admin_port + + def _client(self, **kwargs): + from keystoneclient.v2_0 import client as ks_client + + kc = ks_client.Client(endpoint=self._admin_url(), + auth_url=self._public_url(), + **kwargs) + kc.authenticate() + # have to manually overwrite the management url after authentication + kc.management_url = self._admin_url() + return kc def get_client(self, user_ref=None, tenant_ref=None): if user_ref is None: @@ -79,6 +71,10 @@ class KcMasterTestCase(CompatTestCase): password=user_ref['password'], tenant_id=tenant_id) + +class KeystoneClientTests(object): + """Tests for all versions of keystoneclient.""" + def test_authenticate_tenant_name_and_tenants(self): client = self.get_client() tenants = client.tenants.list() @@ -169,33 +165,6 @@ class KcMasterTestCase(CompatTestCase): tenants = client.tenants.list() self.assertEquals(len(tenants), 1) - def test_tenant_add_and_remove_user(self): - client = self.get_client() - client.roles.add_user_to_tenant(tenant_id=self.tenant_baz['id'], - user_id=self.user_foo['id'], - role_id=self.role_useless['id']) - tenant_refs = client.tenants.list() - self.assert_(self.tenant_baz['id'] in - [x.id for x in tenant_refs]) - - # get the "role_refs" so we get the proper id, this is how the clients - # do it - roleref_refs = client.roles.get_user_role_refs( - user_id=self.user_foo['id']) - for roleref_ref in roleref_refs: - if (roleref_ref.roleId == self.role_useless['id'] and - roleref_ref.tenantId == self.tenant_baz['id']): - # use python's scope fall through to leave roleref_ref set - break - - client.roles.remove_user_from_tenant(tenant_id=self.tenant_baz['id'], - user_id=self.user_foo['id'], - role_id=roleref_ref.id) - - tenant_refs = client.tenants.list() - self.assert_(self.tenant_baz['id'] not in - [x.id for x in tenant_refs]) - def test_invalid_password(self): from keystoneclient import exceptions as client_exceptions @@ -280,7 +249,7 @@ class KcMasterTestCase(CompatTestCase): client.roles.delete(role=role.id) self.assertRaises(client_exceptions.NotFound, client.roles.get, - role=test_role) + role=role.id) def test_role_list(self): client = self.get_client() @@ -288,11 +257,6 @@ class KcMasterTestCase(CompatTestCase): # TODO(devcamcar): This assert should be more specific. self.assertTrue(len(roles) > 0) - def test_roles_get_by_user(self): - client = self.get_client() - roles = client.roles.get_user_role_refs(user_id='foo') - self.assertTrue(len(roles) > 0) - def test_ec2_credential_crud(self): client = self.get_client() creds = client.ec2.list(user_id=self.user_foo['id']) @@ -433,6 +397,66 @@ class KcMasterTestCase(CompatTestCase): # TODO(ja): determine what else todo -class KcEssex3TestCase(KcMasterTestCase): +class KcMasterTestCase(CompatTestCase, KeystoneClientTests): + def get_checkout(self): + return KEYSTONECLIENT_REPO, 'master' + + def test_tenant_add_and_remove_user(self): + client = self.get_client() + client.roles.add_user_role(tenant=self.tenant_baz['id'], + user=self.user_foo['id'], + role=self.role_useless['id']) + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] in + [x.id for x in tenant_refs]) + + client.roles.remove_user_role(tenant=self.tenant_baz['id'], + user=self.user_foo['id'], + role=self.role_useless['id']) + + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] not in + [x.id for x in tenant_refs]) + + def test_roles_get_by_user(self): + client = self.get_client() + roles = client.roles.roles_for_user(user=self.user_foo['id'], + tenant=self.tenant_bar['id']) + self.assertTrue(len(roles) > 0) + + +class KcEssex3TestCase(CompatTestCase, KeystoneClientTests): def get_checkout(self): return KEYSTONECLIENT_REPO, 'essex-3' + + def test_tenant_add_and_remove_user(self): + client = self.get_client() + client.roles.add_user_to_tenant(tenant_id=self.tenant_baz['id'], + user_id=self.user_foo['id'], + role_id=self.role_useless['id']) + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] in + [x.id for x in tenant_refs]) + + # get the "role_refs" so we get the proper id, this is how the clients + # do it + roleref_refs = client.roles.get_user_role_refs( + user_id=self.user_foo['id']) + for roleref_ref in roleref_refs: + if (roleref_ref.roleId == self.role_useless['id'] and + roleref_ref.tenantId == self.tenant_baz['id']): + # use python's scope fall through to leave roleref_ref set + break + + client.roles.remove_user_from_tenant(tenant_id=self.tenant_baz['id'], + user_id=self.user_foo['id'], + role_id=roleref_ref.id) + + tenant_refs = client.tenants.list() + self.assert_(self.tenant_baz['id'] not in + [x.id for x in tenant_refs]) + + def test_roles_get_by_user(self): + client = self.get_client() + roles = client.roles.get_user_role_refs(user_id='foo') + self.assertTrue(len(roles) > 0) From 0b34e5f511c8dc57d11ddef874fdbd2ea7bb6ff5 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 11:49:21 -0800 Subject: [PATCH 278/334] deal with tags in git checkout --- keystone/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/keystone/test.py b/keystone/test.py index 9f2773f7e8..720b4db832 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -40,6 +40,7 @@ def testsdir(*p): def checkout_vendor(repo, rev): + # TODO(termie): this function is a good target for some optimizations :PERF name = repo.split('/')[-1] if name.endswith('.git'): name = name[:-4] @@ -57,7 +58,8 @@ def checkout_vendor(repo, rev): utils.git('clone', repo, revdir) cd(revdir) - utils.git('pull') + utils.git('checkout', '-q', 'master') + utils.git('pull', '-q') utils.git('checkout', '-q', rev) # write out a modified time From 37e1c5c57a140c41b154ba9901855026e4ee5c3b Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 14:49:50 -0800 Subject: [PATCH 279/334] don't automatically parse sys.argv for cfg --- keystone/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keystone/config.py b/keystone/config.py index 2d69d7a644..5062927a54 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -15,6 +15,7 @@ class ConfigMixin(object): def __call__(self, config_files=None, *args, **kw): if config_files is not None: self._opts['config_file']['opt'].default = config_files + kw.setdefault('args', []) return super(ConfigMixin, self).__call__(*args, **kw) def __getitem__(self, key, default=None): From 09bd758fb88387cf9e58666b1bbcded4e019037e Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 14:50:02 -0800 Subject: [PATCH 280/334] deal with reparsing the config files --- tests/test_cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 82fde86df8..e66d6f636e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,8 +15,15 @@ KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): def setUp(self): super(CliMasterTestCase, self).setUp() + # NOTE(termie): we need to reset and reparse the config here because + # cli adds new command-line config options + # NOTE(termie): we are importing cli here because it imports + # keystoneclient, which we are loading from different + # sources between tests + CONF.reset() from keystone import cli self.cli = cli + self.config() def get_client(self, user_ref=None, tenant_ref=None): if user_ref is None: From 3cfea52ace7d273f77d4bc58e095f20030f8c48b Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 15:53:26 -0800 Subject: [PATCH 281/334] accept POST or PUT for tenant update this is a keystone legacy issue --- keystone/contrib/admin_crud/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/contrib/admin_crud/core.py b/keystone/contrib/admin_crud/core.py index 25be6b399f..ce0867e711 100644 --- a/keystone/contrib/admin_crud/core.py +++ b/keystone/contrib/admin_crud/core.py @@ -25,7 +25,7 @@ class CrudExtension(wsgi.ExtensionRouter): mapper.connect("/tenants/{tenant_id}", controller=tenant_controller, action="update_tenant", - conditions=dict(method=["PUT"])) + conditions=dict(method=["PUT", "POST"])) mapper.connect("/tenants/{tenant_id}", controller=tenant_controller, action="delete_tenant", From a703983713306278ecee1c7cf993a5c5772e1035 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 15:56:09 -0800 Subject: [PATCH 282/334] skip the two tests where testing code is failing the cli is rather well tested already from the other tests, these tests are only failing because the testing code is having trouble with multiple copies of the keystoneclient.exceptions.NotFound error (the tests pass when run individually) and I am no longer willing to waste time trying to fix it --- tests/test_cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index e66d6f636e..5d06d49185 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,8 +61,15 @@ class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): def test_tenant_create_update_and_delete(self): raise nose.exc.SkipTest('cli does not support booleans yet') + def test_invalid_password(self): raise nose.exc.SkipTest('N/A') def test_user_create_update_delete(self): raise nose.exc.SkipTest('cli does not support booleans yet') + + def test_role_create_and_delete(self): + raise nose.exc.SkipTest('cli testing code does not handle 404 well') + + def test_service_create_and_delete(self): + raise nose.exc.SkipTest('cli testing code does not handle 404 well') From 40525e0e52966454e75a4ba9ed689280caaaf61d Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 1 Feb 2012 16:51:46 -0800 Subject: [PATCH 283/334] be more safe with getting json aprams --- keystone/middleware/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 21e0f624c6..46d38673b8 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -89,9 +89,13 @@ class JsonBodyMiddleware(wsgi.Middleware): if not params_json: return - params_parsed = json.loads(params_json) - if not params_parsed: - params_parsed = {} + params_parsed = {} + try: + params_parsed = json.loads(params_json) + finally: + if not params_parsed: + params_parsed = {} + params = {} for k, v in params_parsed.iteritems(): if k in ('self', 'context'): From 69bb042a86a73773044b94437fe17a1639ec7e79 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Thu, 2 Feb 2012 09:34:16 -0500 Subject: [PATCH 284/334] Renamed keystone-server to keystone-all based on comments in LP: #910484. Signed-off-by: Chuck Short --- bin/{keystone-server => keystone-all} | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename bin/{keystone-server => keystone-all} (100%) diff --git a/bin/keystone-server b/bin/keystone-all similarity index 100% rename from bin/keystone-server rename to bin/keystone-all diff --git a/setup.py b/setup.py index fa5c3bea6d..127867a8b0 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup(name='keystone', author_email='openstack@lists.launchpad.net', url='http://www.openstack.org', packages=find_packages(exclude=['test', 'bin']), - scripts=['bin/keystone-server', 'bin/keystone-manage'], + scripts=['bin/keystone-all', 'bin/keystone-manage'], zip_safe=False, install_requires=['setuptools', 'python-keystoneclient'], ) From 0027f90c0e22b0adfa78d534658e56c72f81dd55 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Thu, 2 Feb 2012 09:39:02 -0500 Subject: [PATCH 285/334] Missed one more keystone-server. Signed-off-by: Chuck Short --- bin/keystone-all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/keystone-all b/bin/keystone-all index d30d9c6192..7645a949db 100755 --- a/bin/keystone-all +++ b/bin/keystone-all @@ -12,7 +12,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) if os.path.exists(os.path.join(possible_topdir, - 'keystone-server', + 'keystone-all', '__init__.py')): sys.path.insert(0, possible_topdir) From 433e7db49920d5b0d233d623d7376589c2897d57 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Thu, 2 Feb 2012 08:48:39 -0800 Subject: [PATCH 286/334] minor docstring update for new locations --- keystone/catalog/core.py | 2 +- keystone/contrib/ec2/core.py | 2 +- keystone/identity/core.py | 2 +- keystone/policy/core.py | 2 +- keystone/token/core.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index aecae36adf..93c0de2077 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -17,7 +17,7 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the Catalog backend. - See :mod:`keystone.manager.Manager` for more details on how this + See :mod:`keystone.common.manager.Manager` for more details on how this dynamically calls the backend. """ diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 08e46647bc..0d7725a164 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -39,7 +39,7 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the EC2 Credentials backend. - See :mod:`keystone.manager.Manager` for more details on how this + See :mod:`keystone.common.manager.Manager` for more details on how this dynamically calls the backend. """ diff --git a/keystone/identity/core.py b/keystone/identity/core.py index e841898545..c418bce10d 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -22,7 +22,7 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the Identity backend. - See :mod:`keystone.manager.Manager` for more details on how this + See :mod:`keystone.common.manager.Manager` for more details on how this dynamically calls the backend. """ diff --git a/keystone/policy/core.py b/keystone/policy/core.py index d5af09fe08..694f62853b 100644 --- a/keystone/policy/core.py +++ b/keystone/policy/core.py @@ -12,7 +12,7 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the Policy backend. - See :mod:`keystone.manager.Manager` for more details on how this + See :mod:`keystone.common.manager.Manager` for more details on how this dynamically calls the backend. """ diff --git a/keystone/token/core.py b/keystone/token/core.py index b0385f5a60..e78276969f 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -12,7 +12,7 @@ CONF = config.CONF class Manager(manager.Manager): """Default pivot point for the Token backend. - See :mod:`keystone.manager.Manager` for more details on how this + See :mod:`keystone.common.manager.Manager` for more details on how this dynamically calls the backend. """ From 446b26850d1afa1bd239c3048a23fc818b86c8f0 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 2 Feb 2012 16:58:32 -0800 Subject: [PATCH 287/334] use our own logging module --- keystone/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/config.py b/keystone/config.py index 5062927a54..09c18ca6ac 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -1,11 +1,11 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 import gettext -import logging import sys import os from keystone.common import cfg +from keystone.common import logging gettext.install('keystone', unicode=1) From 4f651ba2425fa418b101a266a5b73d758d1b3d04 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Thu, 2 Feb 2012 20:57:45 -0800 Subject: [PATCH 288/334] updating tox.ini with test pip requirements --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e9a2fb389b..96c4e0da9f 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = py26,py27,pep8 [testenv] -deps = -r{toxinidir}/tools/pip-requires +deps = -r{toxinidir}/tools/pip-requires-test commands = nosetests [testenv:pep8] From 32ff03b6dc9d8a6b5c1102faabf7f84bd1ebd3c1 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Thu, 2 Feb 2012 20:40:39 -0800 Subject: [PATCH 289/334] updating docs: * reference keystone-all instead of keystone * remove reference to keystone.version * rename gnerated man page * spacing and line wrapping --- README.rst | 4 +- docs/source/conf.py | 16 +- docs/source/configuration.rst | 154 +++++++++++++++--- docs/source/configuringservices.rst | 23 ++- docs/source/developing.rst | 30 +++- .../man/{keystone.rst => keystone-all.rst} | 12 +- docs/source/nova-api-paste.rst | 3 +- docs/source/setup.rst | 17 +- 8 files changed, 192 insertions(+), 67 deletions(-) rename docs/source/man/{keystone.rst => keystone-all.rst} (88%) diff --git a/README.rst b/README.rst index 248178b079..2d24892aeb 100644 --- a/README.rst +++ b/README.rst @@ -152,7 +152,7 @@ Approach to Authorization (Policy) Various components in the system require that different actions are allowed based on whether the user is authorized to perform that action. -For the purposes of Keystone Light there are only a couple levels of +For the purposes of Keystone there are only a couple levels of authorization being checked for: * Require that the performing user is considered an admin. @@ -160,7 +160,7 @@ authorization being checked for: Other systems wishing to use the policy engine will require additional styles of checks and will possibly write completely custom backends. Backends included -in Keystone Light are: +in Keystone are: Trivial True diff --git a/docs/source/conf.py b/docs/source/conf.py index edae0b8a16..fc7d94766b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -52,7 +52,7 @@ master_doc = 'index' # General information about the project. project = u'keystone' -copyright = u'2012, termie' +copyright = u'2012, OpenStack, LLC' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -105,7 +105,7 @@ modindex_common_prefix = ['keystone.'] man_pages = [ ('man/keystone-manage', 'keystone-manage', u'Keystone Management Utility', [u'OpenStack'], 1), - ('man/keystone', 'keystone', u'Keystone Startup Command', + ('man/keystone-all', 'keystone-all', u'Keystone Startup Command', [u'OpenStack'], 1), ] @@ -207,8 +207,8 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'keystone.tex', u'keystone Documentation', - u'termie', 'manual'), + ('index', 'keystone.tex', u'Keystone Documentation', + u'OpenStack', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -237,8 +237,8 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'keystone', u'keystone Documentation', - [u'termie'], 1) + ('index', 'keystone', u'Keystone Documentation', + [u'OpenStack'], 1) ] # If true, show URL addresses after external links. @@ -251,8 +251,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'keystone', u'keystone Documentation', - u'termie', 'keystone', 'One line description of project.', + ('index', 'keystone', u'Keystone Documentation', + u'OpenStack', 'keystone', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 782e9fb572..7fbcf0261b 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -22,6 +22,7 @@ Configuring Keystone :maxdepth: 1 man/keystone-manage + man/keystone-all Once Keystone is installed, it is configured via a primary configuration file (``etc/keystone.conf``), possibly a separate logging configuration file, and @@ -34,15 +35,14 @@ Keystone Configuration File The keystone configuration file is an 'ini' file format with sections, extended from Paste_, a common system used to configure python WSGI based applications. In addition to the paste config entries, general configuration -values are stored under [DEFAULT] and [sql], and then drivers for the various -backend components are included under their individual sections. +values are stored under ``[DEFAULT]``, ``[sql]``, ``[ec2]`` and then drivers +for the various services are included under their individual sections. -The driver sections include: +The services include: * ``[identity]`` - the python module that backends the identity system * ``[catalog]`` - the python module that backends the service catalog * ``[token]`` - the python module that backends the token providing mechanisms * ``[policy]`` - the python module that drives the policy system for RBAC -* ``[ec2]`` - the python module providing the EC2 translations for OpenStack The keystone configuration file is expected to be named ``keystone.conf``. When starting up Keystone, you can specify a different configuration file to @@ -61,7 +61,8 @@ file under ``log_config``. If you wish to route all your logging through syslog, there is a ``use_syslog`` option also in the [DEFAULT] section that easy. -A sample logging file is available with the project in the directory ``etc/logging.conf.sample``. Like other OpenStack projects, keystone uses the +A sample logging file is available with the project in the directory +``etc/logging.conf.sample``. Like other OpenStack projects, keystone uses the `python logging module`, which includes extensive configuration options for choosing the output levels and formats. @@ -78,6 +79,18 @@ Sample Configuration Files * ``etc/keystone.conf`` * ``etc/logging.conf.sample`` +Running Keystone +================ + +Running keystone is simply starting the services by using the command:: + + keystone-all + +Invoking this command starts up two wsgi.Server instances, configured by the +``keystone.conf`` file as described above. One of these wsgi 'servers' is +``admin`` (the administration API) and the other is ``main`` (the +primary/public API interface). Both of these run in a single process. + Initializing Keystone ===================== @@ -129,7 +142,8 @@ containers within Swift. A tenant can have zero or more users, Users can be asso keyword arguments * tenant_name -* id (optional) +* description (optional, defaults to None) +* enabled (optional, defaults to True) example:: @@ -142,7 +156,7 @@ creates a tenant named "admin". keyword arguments -* tenant_id +* tenant example:: @@ -153,9 +167,10 @@ example:: keyword arguments -* description -* name * tenant_id +* tenant_name (optional, defaults to None) +* description (optional, defaults to None) +* enabled (optional, defaults to True) example:: @@ -175,6 +190,8 @@ keyword arguments * name * password * email +* tenant_id (optional, defaults to None) +* enabled (optional, defaults to True) example:: @@ -188,31 +205,56 @@ example:: keyword arguments +* user + +example:: + + keystone-manage user --ks-id-only delete f2b7b39c860840dfa47d9ee4adffa0b3 + ``user list`` ^^^^^^^^^^^^^ +list users in the system, optionally by a specific tenant (identified by tenant_id) + keyword arguments +* tenant_id (optional, defaults to None) +* limit (optional, defaults to None) +* marker (optional, defaults to None) + ``user update_email`` ^^^^^^^^^^^^^^^^^^^^^ keyword arguments +* user +* email + ``user update_enabled`` ^^^^^^^^^^^^^^^^^^^^^^^ keyword arguments +* user +* enabled (True or False) + ``user update_password`` ^^^^^^^^^^^^^^^^^^^^^^^^ keyword arguments +* user +* password + ``user update_tenant`` ^^^^^^^^^^^^^^^^^^^^^^ keyword arguments +* user +* tenant + + Roles ----- @@ -227,27 +269,67 @@ exmaple:: keystone-manage role --ks-id-only create name=Admin -``role add_user_to_tenant`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``role delete`` +^^^^^^^^^^^^^^^ keyword arguments -* role_id -* user_id -* tenant_id +* role + +exmaple:: + + keystone-manage role delete role=19d1d3344873464d819c45f521ff9890 + +``role list`` +^^^^^^^^^^^^^^^ + +exmaple:: + + keystone-manage role list + +``role add_user_role`` +^^^^^^^^^^^^^^^^^^^^^^ + +keyword arguments + +* role +* user +* tenant (optional, defaults to None) example:: - keystone-manage role add_user_to_tenant \ - role_id=19d1d3344873464d819c45f521ff9890 \ - user_id=08741d8ed88242ca88d1f61484a0fe3b \ - tenant_id=20601a7f1d94447daa4dff438cb1c209 + keystone-manage role add_user_role \ + role=19d1d3344873464d819c45f521ff9890 \ + user=08741d8ed88242ca88d1f61484a0fe3b \ + tenant=20601a7f1d94447daa4dff438cb1c209 -``role remove_user_from_tenant`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``role remove_user_role`` +^^^^^^^^^^^^^^^^^^^^^^^^^ -``role get_user_role_refs`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +keyword arguments + +* role +* user +* tenant (optional, defaults to None) + +example:: + + keystone-manage role remove_user_to_tenant \ + role=19d1d3344873464d819c45f521ff9890 \ + user=08741d8ed88242ca88d1f61484a0fe3b \ + tenant=20601a7f1d94447daa4dff438cb1c209 + +``role roles_for_user`` +^^^^^^^^^^^^^^^^^^^^^^^ + +keyword arguments + +* user +* tenant (optional, defaults to None) + +example:: + + keystone-manage role roles_for_user user=08741d8ed88242ca88d1f61484a0fe3b Services -------- @@ -267,3 +349,31 @@ example:: name=nova \ service_type=compute \ description="Nova Compute Service" + +``service list`` +^^^^^^^^^^^^^^^^ + +keyword arguments + +example:: + + keystone-manage service list + +``service get`` +^^^^^^^^^^^^^^^ + +keyword arguments + +example:: + + keystone-manage service get id=08741d8ed88242ca88d1f61484a0fe3b + +``service delete`` +^^^^^^^^^^^^^^^^^^ + +keyword arguments + +example:: + + keystone-manage service delete id=08741d8ed88242ca88d1f61484a0fe3b + diff --git a/docs/source/configuringservices.rst b/docs/source/configuringservices.rst index 88bb9c15db..5ea75461bb 100644 --- a/docs/source/configuringservices.rst +++ b/docs/source/configuringservices.rst @@ -24,12 +24,14 @@ Configuring Services to work with Keystone nova-api-paste middleware_architecture -Once Keystone is installed and running (see :doc:`configuration`), services need to be configured to work -with it. To do this, we primarily install and configure middleware for the OpenStack service to handle authentication tasks or otherwise interact with Keystone. +Once Keystone is installed and running (see :doc:`configuration`), services +need to be configured to work with it. To do this, we primarily install and +configure middleware for the OpenStack service to handle authentication tasks +or otherwise interact with Keystone. In general: -* Clients making calls to the service will pass in an authentication token. -* The Keystone middleware will look for and validate that token, taking the appropriate action. +* Clients making calls to the service will pass in an authentication token. +* The Keystone middleware will look for and validate that token, taking the appropriate action. * It will also retrive additional information from the token such as user name, id, tenant name, id, roles, etc... The middleware will pass those data down to the service as headers. More details on the architecture of @@ -42,13 +44,16 @@ Admin Token ----------- For a default installation of Keystone, before you can use the REST API, you -need to define an authorization token. This is configured in the keystone.conf file under the section ``[DEFAULT]``. In the sample file provided with the keystone project, the line defining this token is +need to define an authorization token. This is configured in ``keystone.conf`` +file under the section ``[DEFAULT]``. In the sample file provided with the keystone project, the line defining this token is [DEFAULT] admin_token = ADMIN -This is a "shared secret" between keystone and other openstack services, and will need to be set the -same between those services in order for keystone services to function correctly. +This configured token is a "shared secret" between keystone and other +openstack services (for example: nova, swift, glance, or horizon), and will +need to be set the same between those services in order for keystone services +to function correctly. Setting up tenants, users, and roles ------------------------------------ @@ -58,7 +63,7 @@ You need to minimally define a tenant, user, and role to link the tenant and use Setting up services =================== -Defining Services +Defining Services ----------------- Keystone also acts as a service catalog to let other OpenStack systems know @@ -146,7 +151,7 @@ rather than it's built in 'tempauth'. Note that the optional "cache" property in the keystone filter allows any service (not just Swift) to register its memcache client in the WSGI - environment. If such a cache exists, Keystone middleware will utilize it + environment. If such a cache exists, Keystone middleware will utilize it to store validated token information, which could result in better overall performance. diff --git a/docs/source/developing.rst b/docs/source/developing.rst index f529761eb0..f7c0b87bf9 100644 --- a/docs/source/developing.rst +++ b/docs/source/developing.rst @@ -22,7 +22,7 @@ Contributing Code ================= To contribute code, sign up for a Launchpad account and sign a contributor license agreement, -available on the ``_. Once the CLA is signed you +available on the ``_. Once the CLA is signed you can contribute code through the Gerrit version control system which is related to your Launchpad account. To contribute tests, docs, code, etc, refer to our `Gerrit-Jenkins-Github Workflow`_. @@ -40,8 +40,8 @@ Running Keystone To run the keystone Admin and API server instances, use:: - $ tools/with_venv.sh bin/keystone - + $ tools/with_venv.sh bin/keystone-all + this runs keystone with the configuration the etc/ directory of the project. See :doc:`configuration` for details on how Keystone is configured. Interacting with Keystone @@ -76,20 +76,32 @@ light integration testing to verify the keystone API against other projects. Test Structure -------------- -``./run_test.sh`` uses it's python cohort (``run_tests.py``) to iterate through the ``tests`` directory, using Nosetest to collect the tests and invoke them using an -OpenStack custom test running that displays the tests as well as the time taken to +``./run_test.sh`` uses its python cohort (``run_tests.py``) to iterate +through the ``tests`` directory, using Nosetest to collect the tests and +invoke them using an OpenStack custom test running that displays the tests +as well as the time taken to run those tests. -Within the tests directory, the general structure of the tests is a basic set of tests represented under a test class, and then subclasses of those tests under other classes with different configurations to drive different backends through the APIs. +Within the tests directory, the general structure of the tests is a basic +set of tests represented under a test class, and then subclasses of those +tests under other classes with different configurations to drive different +backends through the APIs. -For example, ``test_backend.py`` has a sequence of tests under the class ``IdentityTests`` that will work with the default drivers as configured in this projects etc/ directory. ``test_backend_sql.py`` subclasses those tests, changing the configuration by overriding with configuration files stored in the tests directory aimed at enabling the SQL backend for the Identity module. +For example, ``test_backend.py`` has a sequence of tests under the class +``IdentityTests`` that will work with the default drivers as configured in +this projects etc/ directory. ``test_backend_sql.py`` subclasses those tests, +changing the configuration by overriding with configuration files stored in +the tests directory aimed at enabling the SQL backend for the Identity module. -Likewise, ``test_cli.py`` takes advantage of the tests written aainst ``test_keystoneclient`` to verify the same tests function through different drivers. +Likewise, ``test_cli.py`` takes advantage of the tests written aainst +``test_keystoneclient`` to verify the same tests function through different +drivers. Testing Schema Migrations ------------------------- -The application of schema migrations can be tested using SQLAlchemy Migrate’s built-in test runner, one migration at a time. +The application of schema migrations can be tested using SQLAlchemy Migrate’s +built-in test runner, one migration at a time. .. WARNING:: diff --git a/docs/source/man/keystone.rst b/docs/source/man/keystone-all.rst similarity index 88% rename from docs/source/man/keystone.rst rename to docs/source/man/keystone-all.rst index 74d0bb69e1..66d0ee4e0d 100644 --- a/docs/source/man/keystone.rst +++ b/docs/source/man/keystone-all.rst @@ -16,17 +16,19 @@ Keystone Management Utility SYNOPSIS ======== - keystone [options] + keystone-all [options] DESCRIPTION =========== -keystone starts both the service and administrative APIs for Keystone. +keystone-all starts both the service and administrative APIs in a single +process to provide catalog, authorization, and authentication services for +OpenStack. USAGE ===== - keystone ``keystone [options]`` + ``keystone-all [options]`` Common Options: ^^^^^^^^^^^^^^^ @@ -63,7 +65,7 @@ programs.:: syslog (defaults to LOG_USER) --use-syslog Use syslog for logging. --nouse-syslog Use syslog for logging. - + FILES ===== @@ -77,5 +79,5 @@ SEE ALSO SOURCE ====== -* Keystone is sourced in GitHub `Keystone `__ +* Keystone source in managed in GitHub `Keystone `__ * Keystone bugs are managed at Launchpad `Launchpad Keystone `__ diff --git a/docs/source/nova-api-paste.rst b/docs/source/nova-api-paste.rst index 586bac72ed..602895ac1e 100644 --- a/docs/source/nova-api-paste.rst +++ b/docs/source/nova-api-paste.rst @@ -136,7 +136,8 @@ nova-api-paste example auth_host = 127.0.0.1 auth_port = 35357 auth_protocol = http - auth_uri = http://127.0.0.1:5000/ + auth_uri = http://your_keystone_host.com:5000/ + ;identical to the admin token defined in keystone.conf admin_token = 999888777666 ;Uncomment next line and check ip:port to use memcached to cache token requests ;memcache_hosts = 127.0.0.1:11211 diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 4f6b6137c6..7d03fd6669 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -62,10 +62,10 @@ When that is complete, you can:: Installing dependencies ======================= -Keystone maintains two lists of dependencies: +Keystone maintains two lists of dependencies:: tools/pip-requires - tools/pip-requires-test + tools/pip-requires-test The first is the list of dependencies needed for running keystone, the second list includes dependencies used for active development and testing of keystone itself. @@ -97,7 +97,7 @@ PyPi Packages and VirtualEnv ---------------------------- We recommend establishing a virtualenv to run keystone within. Virtualenv limits the python environment -to just what you're installing as depdendencies, useful to keep a clean environment for working on +to just what you're installing as depdendencies, useful to keep a clean environment for working on Keystone. The tools directory in keystone has a script already created to make this very simple:: $ python tools/install_venv.py @@ -125,10 +125,10 @@ into your system from the requires files:: # Install the dependencies for developing, testing, and running keystone $ pip install -r tools/pip-requires-test - + # Fake-install the project by symlinking Keystone into your Python site-packages $ python setup.py develop - + Verifying Keystone is set up ============================ @@ -142,14 +142,9 @@ the libraries. If you're using a virtualenv, don't forget to activate it:: You should then be able to `import keystone` from your Python shell without issue:: - >>> import keystone.version + >>> import keystone >>> -If you want to check the version of Keystone you are running: - - >>> print keystone.version.version() - 2012.1-dev - If you can import keystone successfully, you should be ready to move on to :doc:`developing` Troubleshooting From cc371272f2f650765f39d03bd17c1afc80b0e561 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 5 Feb 2012 20:32:37 +0000 Subject: [PATCH 290/334] ran through all commands to verify keywords against current (master) keystonelight --- docs/source/configuration.rst | 58 ++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 7fbcf0261b..d4ea886868 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -195,7 +195,7 @@ keyword arguments example:: - keystone-manage user --ks-id-only create \ + keystone-manage user --id-only create \ name=admin \ password=secrete \ email=admin@example.com @@ -209,7 +209,7 @@ keyword arguments example:: - keystone-manage user --ks-id-only delete f2b7b39c860840dfa47d9ee4adffa0b3 + keystone-manage user delete user=f2b7b39c860840dfa47d9ee4adffa0b3 ``user list`` ^^^^^^^^^^^^^ @@ -219,8 +219,10 @@ list users in the system, optionally by a specific tenant (identified by tenant_ keyword arguments * tenant_id (optional, defaults to None) -* limit (optional, defaults to None) -* marker (optional, defaults to None) + +example:: + + keystone-manage user list ``user update_email`` ^^^^^^^^^^^^^^^^^^^^^ @@ -230,6 +232,11 @@ keyword arguments * user * email +example:: + + keystone-manage user update_email user=03c84b51574841ba9a0d8db7882ac645 email="someone@somewhere.com" + + ``user update_enabled`` ^^^^^^^^^^^^^^^^^^^^^^^ @@ -238,6 +245,10 @@ keyword arguments * user * enabled (True or False) +example:: + + keystone-manage user update_enabled user=03c84b51574841ba9a0d8db7882ac645 enabled=False + ``user update_password`` ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -246,6 +257,10 @@ keyword arguments * user * password +example:: + + keystone-manage user update_password user=03c84b51574841ba9a0d8db7882ac645 password=foo + ``user update_tenant`` ^^^^^^^^^^^^^^^^^^^^^^ @@ -254,6 +269,21 @@ keyword arguments * user * tenant +example:: + + keystone-manage user update_tenant user=03c84b51574841ba9a0d8db7882ac645 tenant=b7b8be32c4be4208949f0373c5909e3b + +``user get`` +^^^^^^^^^^^^ + +keyword arguments + +* user + +example:: + + keystone-manage ususer get user=03c84b51574841ba9a0d8db7882ac645 + Roles ----- @@ -267,7 +297,7 @@ keyword arguments exmaple:: - keystone-manage role --ks-id-only create name=Admin + keystone-manage role --id-only create name=Admin ``role delete`` ^^^^^^^^^^^^^^^ @@ -287,6 +317,18 @@ exmaple:: keystone-manage role list +``role get`` +^^^^^^^^^^^^ + +keysword arguments + +* role + +exmaple:: + + keystone-manage role get role=19d1d3344873464d819c45f521ff9890 + + ``role add_user_role`` ^^^^^^^^^^^^^^^^^^^^^^ @@ -294,13 +336,13 @@ keyword arguments * role * user -* tenant (optional, defaults to None) +* tenant example:: keystone-manage role add_user_role \ - role=19d1d3344873464d819c45f521ff9890 \ - user=08741d8ed88242ca88d1f61484a0fe3b \ + role=3a751f78ef4c412b827540b829e2d7dd \ + user=03c84b51574841ba9a0d8db7882ac645 \ tenant=20601a7f1d94447daa4dff438cb1c209 ``role remove_user_role`` From 6a5c5248a76a561d65bdc1868408c4c7cb27e934 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Mon, 6 Feb 2012 09:28:53 -0600 Subject: [PATCH 291/334] Added support for DELETE /tokens/{token_id} --- keystone/service.py | 12 ++++++++++++ tests/test_cli.py | 3 +++ tests/test_keystoneclient.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/keystone/service.py b/keystone/service.py index 05cca4e600..8e1f056e77 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -32,6 +32,10 @@ class AdminRouter(wsgi.ComposingRouter): controller=auth_controller, action='validate_token', conditions=dict(method=['GET'])) + mapper.connect('/tokens/{token_id}', + controller=auth_controller, + action='delete_token', + conditions=dict(method=['DELETE'])) mapper.connect('/tokens/{token_id}/endpoints', controller=auth_controller, action='endpoints', @@ -255,6 +259,14 @@ class TokenController(wsgi.Application): roles_ref.append(self.identity_api.get_role(context, role_id)) return self._format_token(token_ref, roles_ref) + def delete_token(self, context, token_id): + """Delete a token, effectively invalidating it for authz.""" + # TODO(termie): this stuff should probably be moved to middleware + self.assert_admin(context) + + token_ref = self.token_api.delete_token(context=context, + token_id=token_id) + def endpoints(self, context, token_id): """Return service catalog endpoints.""" token_ref = self.token_api.get_token(context=context, diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d06d49185..bd4ead735b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -59,6 +59,9 @@ class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): def test_authenticate_token_tenant_name(self): raise nose.exc.SkipTest('N/A') + def test_authenticate_and_delete_token(self): + raise nose.exc.SkipTest('N/A') + def test_tenant_create_update_and_delete(self): raise nose.exc.SkipTest('cli does not support booleans yet') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 1cda1bc620..a32a3b143c 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -122,6 +122,22 @@ class KeystoneClientTests(object): token_client = self._client(token=token, tenant_name='BAR') tenants = token_client.tenants.list() self.assertEquals(tenants[0].id, self.tenant_bar['id']) + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + def test_authenticate_and_delete_token(self): + client = self.get_client() + token = client.auth_token + token_client = self._client(token=token) + tenants = token_client.tenants.list() + self.assertEquals(tenants[0].id, self.tenant_bar['id']) + + client.tokens.delete(token_client.auth_token) + + # FIXME(dolph): this should raise unauthorized + # from keystoneclient import exceptions as client_exceptions + # with self.assertRaises(client_exceptions.Unauthorized): + with self.assertRaises(Exception): + token_client.tenants.list() # TODO(termie): I'm not really sure that this is testing much def test_endpoints(self): @@ -460,3 +476,6 @@ class KcEssex3TestCase(CompatTestCase, KeystoneClientTests): client = self.get_client() roles = client.roles.get_user_role_refs(user_id='foo') self.assertTrue(len(roles) > 0) + + def test_authenticate_and_delete_token(self): + raise nose.exc.SkipTest('N/A') From fca3e9c0453bc6adeb50c77dd1be79933132fdaf Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 5 Feb 2012 16:24:42 -0800 Subject: [PATCH 292/334] adding a token service Driver to define the interface --- keystone/token/backends/kvs.py | 3 ++- keystone/token/core.py | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py index b7c25fb7bf..c19bcc796e 100644 --- a/keystone/token/backends/kvs.py +++ b/keystone/token/backends/kvs.py @@ -1,9 +1,10 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +from keystone import token from keystone.common import kvs -class Token(kvs.Base): +class Token(kvs.Base, token.Driver): # Public interface def get_token(self, token_id): return self.db.get('token-%s' % token_id) diff --git a/keystone/token/core.py b/keystone/token/core.py index e78276969f..47dadcea48 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -19,3 +19,50 @@ class Manager(manager.Manager): def __init__(self): super(Manager, self).__init__(CONF.token.driver) + + +class Driver(object): + """Interface description for a Token driver.""" + + def get_token(self, token_id): + """Get a token by id. + + :param token_id: identity of the token + :type token_id: string + :returns: token_ref or None. + + """ + raise NotImplementedError() + + def create_token(self, token_id, data): + """Create a token by id and data. + + :param token_id: identity of the token + :type token_id: string + :param data: dictionary with additional reference information + + :: + + { + expires='' + id=token_id, + user=user_ref, + tenant=tenant_ref, + metadata=metadata_ref + } + + :type data: dict + :returns: token_ref or None. + + """ + raise NotImplementedError() + + def delete_token(self, token_id): + """Deletes a token by id. + + :param token_id: identity of the token + :type token_id: string + :returns: None. + + """ + raise NotImplementedError() From e0afc0dc327030d34c1fbd7806f555f69e406144 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Mon, 6 Feb 2012 14:28:35 -0600 Subject: [PATCH 293/334] Removed unused reference --- keystone/service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/keystone/service.py b/keystone/service.py index 8e1f056e77..eae882d017 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -264,8 +264,7 @@ class TokenController(wsgi.Application): # TODO(termie): this stuff should probably be moved to middleware self.assert_admin(context) - token_ref = self.token_api.delete_token(context=context, - token_id=token_id) + self.token_api.delete_token(context=context, token_id=token_id) def endpoints(self, context, token_id): """Return service catalog endpoints.""" From 62a92c44baf533fe316016c1fa5be892fc85009c Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 6 Feb 2012 20:43:24 +0000 Subject: [PATCH 294/334] fixing grammar, noting broken enable, adding hacking with prefs for project --- HACKING.rst | 189 ++++++++++++++++++++++++++++ docs/source/configuration.rst | 11 ++ docs/source/configuringservices.rst | 42 +++++-- docs/source/man/keystone-all.rst | 2 +- 4 files changed, 231 insertions(+), 13 deletions(-) create mode 100644 HACKING.rst diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000000..a0d052a807 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,189 @@ +Keystone Style Commandments +=========================== + +- Step 1: Read http://www.python.org/dev/peps/pep-0008/ +- Step 2: Read http://www.python.org/dev/peps/pep-0008/ again +- Step 3: Read on + + +General +------- +- Put two newlines between top-level code (funcs, classes, etc) +- Put one newline between methods in classes and anywhere else +- Do not write "except:", use "except Exception:" at the very least +- Include your name with TODOs as in "#TODO(termie)" +- Do not name anything the same name as a built-in or reserved word + +TODO vs FIXME +------------- + +- TODO(name): implies that something should be done (cleanup, refactoring, + etc), but is expected to be functional. +- FIXME(name): implies that the method/function/etc shouldn't be used until + that code is resolved and bug fixed. + +Imports +------- +- Do not import objects, only modules +- Do not import more than one module per line +- Do not make relative imports +- Order your imports by the full module path +- Organize your imports according to the following template + +Example:: + + # vim: tabstop=4 shiftwidth=4 softtabstop=4 + {{stdlib imports in human alphabetical order}} + \n + {{third-party lib imports in human alphabetical order}} + \n + {{nova imports in human alphabetical order}} + \n + \n + {{begin your code}} + + +Human Alphabetical Order Examples +--------------------------------- +Example:: + + import httplib + import logging + import random + import StringIO + import time + import unittest + + import eventlet + import webob.exc + + import nova.api.ec2 + from nova.api import openstack + from nova.auth import users + import nova.flags + from nova.endpoint import cloud + from nova import test + + +Docstrings +---------- +Example:: + + """A one line docstring looks like this and ends in a period.""" + + + """A multiline docstring has a one-line summary, less than 80 characters. + + Then a new paragraph after a newline that explains in more detail any + general information about the function, class or method. Example usages + are also great to have here if it is a complex class for function. + + When writing the docstring for a class, an extra line should be placed + after the closing quotations. For more in-depth explanations for these + decisions see http://www.python.org/dev/peps/pep-0257/ + + Describe parameters and return values, using the Sphinx format; the + appropriate syntax is as follows. + + :param foo: the foo parameter + :param bar: the bar parameter + :type bar: parameter type for 'bar' + :returns: return_type -- description of the return value + :returns: description of the return value + :raises: AttributeError, KeyError + """ + + +Dictionaries/Lists +------------------ +If a dictionary (dict) or list object is longer than 80 characters, its items +should be split with newlines. Embedded iterables should have their items +indented. Additionally, the last item in the dictionary should have a trailing +comma. This increases readability and simplifies future diffs. + +Example:: + + my_dictionary = { + "image": { + "name": "Just a Snapshot", + "size": 2749573, + "properties": { + "user_id": 12, + "arch": "x86_64", + }, + "things": [ + "thing_one", + "thing_two", + ], + "status": "ACTIVE", + }, + } + + +Calling Methods +--------------- +Calls to methods 80 characters or longer should format each argument with +newlines. This is not a requirement, but a guideline:: + + unnecessarily_long_function_name('string one', + 'string two', + kwarg1=constants.ACTIVE, + kwarg2=['a', 'b', 'c']) + + +Rather than constructing parameters inline, it is better to break things up:: + + list_of_strings = [ + 'what_a_long_string', + 'not as long', + ] + + dict_of_numbers = { + 'one': 1, + 'two': 2, + 'twenty four': 24, + } + + object_one.call_a_method('string three', + 'string four', + kwarg1=list_of_strings, + kwarg2=dict_of_numbers) + + +Internationalization (i18n) Strings +----------------------------------- +In order to support multiple languages, we have a mechanism to support +automatic translations of exception and log strings. + +Example:: + + msg = _("An error occurred") + raise HTTPBadRequest(explanation=msg) + +If you have a variable to place within the string, first internationalize the +template string then do the replacement. + +Example:: + + msg = _("Missing parameter: %s") % ("flavor",) + LOG.error(msg) + +If you have multiple variables to place in the string, use keyword parameters. +This helps our translators reorder parameters when needed. + +Example:: + + msg = _("The server with id %(s_id)s has no key %(m_key)s") + LOG.error(msg % {"s_id": "1234", "m_key": "imageId"}) + + +Creating Unit Tests +------------------- +For every new feature, unit tests should be created that both test and +(implicitly) document the usage of said feature. If submitting a patch for a +bug that had no unit test, a new passing unit test should be added. If a +submitted bug fix does have a unit test, be sure to add a new one that fails +without the patch and passes with the patch. + +For more information on creating unit tests and utilizing the testing +infrastructure in OpenStack Nova, please read nova/testing/README.rst. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index d4ea886868..8c998700d2 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -200,6 +200,11 @@ example:: password=secrete \ email=admin@example.com +.. warning:: + Until https://bugs.launchpad.net/keystone/+bug/927873 is resolved, the + keystone-manage cli doesn't allow the setting enabled to be False, making + this command partially broken at the moment. + ``user delete`` ^^^^^^^^^^^^^^^ @@ -249,6 +254,12 @@ example:: keystone-manage user update_enabled user=03c84b51574841ba9a0d8db7882ac645 enabled=False +.. warning:: + Until https://bugs.launchpad.net/keystone/+bug/927873 is resolved, the + keystone-manage cli doesn't allow the setting enabled to False, making + this command broken at the moment. + + ``user update_password`` ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/configuringservices.rst b/docs/source/configuringservices.rst index 5ea75461bb..3777ce5ef8 100644 --- a/docs/source/configuringservices.rst +++ b/docs/source/configuringservices.rst @@ -31,11 +31,14 @@ or otherwise interact with Keystone. In general: * Clients making calls to the service will pass in an authentication token. -* The Keystone middleware will look for and validate that token, taking the appropriate action. -* It will also retrive additional information from the token such as user name, id, tenant name, id, roles, etc... +* The Keystone middleware will look for and validate that token, taking the + appropriate action. +* It will also retrive additional information from the token such as user + name, id, tenant name, id, roles, etc... -The middleware will pass those data down to the service as headers. More details on the architecture of -that setup is described in :doc:`middleware_architecture` +The middleware will pass those data down to the service as headers. More +details on the architecture of that setup is described in +:doc:`middleware_architecture` Setting up credentials ====================== @@ -45,7 +48,8 @@ Admin Token For a default installation of Keystone, before you can use the REST API, you need to define an authorization token. This is configured in ``keystone.conf`` -file under the section ``[DEFAULT]``. In the sample file provided with the keystone project, the line defining this token is +file under the section ``[DEFAULT]``. In the sample file provided with the +keystone project, the line defining this token is [DEFAULT] admin_token = ADMIN @@ -58,7 +62,10 @@ to function correctly. Setting up tenants, users, and roles ------------------------------------ -You need to minimally define a tenant, user, and role to link the tenant and user as the most basic set of details to get other services authenticating and authorizing with keystone. See doc:`configuration` for a walk through on how to create tenants, users, and roles. +You need to minimally define a tenant, user, and role to link the tenant and +user as the most basic set of details to get other services authenticating +and authorizing with keystone. See doc:`configuration` for a walk through on +how to create tenants, users, and roles. Setting up services =================== @@ -73,13 +80,24 @@ for the OpenStack Dashboard to properly function. Here's how we define the services:: - keystone-manage service create name=nova service_type=compute description="Nova Compute Service" - keystone-manage service create name=ec2 service_type=ec2 description="EC2 Compatibility Layer" - keystone-manage service create name=glance service_type=image description="Glance Image Service" - keystone-manage service create name=keystone service_type=identity description="Keystone Identity Service" - keystone-manage service create name=swift service_type=object-store description="Swift Service" + keystone-manage service create name=nova \ + service_type=compute \ + description="Nova Compute Service" + keystone-manage service create name=ec2 \ + service_type=ec2 \ + description="EC2 Compatibility Layer" + keystone-manage service create name=glance \ + service_type=image \ + description="Glance Image Service" + keystone-manage service create name=keystone \ + service_type=identity \ + description="Keystone Identity Service" + keystone-manage service create name=swift \ + service_type=object-store \ + description="Swift Service" -The endpoints for these services are defined in a template, an example of which is in the project as the file ``etc/default_catalog.templates``. +The endpoints for these services are defined in a template, an example of +which is in the project as the file ``etc/default_catalog.templates``. Setting Up Middleware ===================== diff --git a/docs/source/man/keystone-all.rst b/docs/source/man/keystone-all.rst index 66d0ee4e0d..fc2d68d7d2 100644 --- a/docs/source/man/keystone-all.rst +++ b/docs/source/man/keystone-all.rst @@ -79,5 +79,5 @@ SEE ALSO SOURCE ====== -* Keystone source in managed in GitHub `Keystone `__ +* Keystone source is managed in GitHub `Keystone `__ * Keystone bugs are managed at Launchpad `Launchpad Keystone `__ From b6a142d8407b441c1023acaeb597036bdb26f3ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 6 Feb 2012 23:01:10 +0000 Subject: [PATCH 295/334] Make ec2 auth actually work --- keystone/cli.py | 1 - keystone/common/utils.py | 30 +++++++++++++++--------------- keystone/contrib/ec2/core.py | 15 ++++++++++++--- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/keystone/cli.py b/keystone/cli.py index e178a31b64..2caab52bc8 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import json import logging -import os import sys import StringIO import textwrap diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 95a1b97103..16692bd635 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -75,17 +75,17 @@ class Ec2Signer(object): def generate(self, credentials): """Generate auth string according to what SignatureVersion is given.""" - if credentials.params['SignatureVersion'] == '0': - return self._calc_signature_0(credentials.params) - if credentials.params['SignatureVersion'] == '1': - return self._calc_signature_1(credentials.params) - if credentials.params['SignatureVersion'] == '2': - return self._calc_signature_2(credentials.params, - credentials.verb, - credentials.host, - credentials.path) + if credentials['params']['SignatureVersion'] == '0': + return self._calc_signature_0(credentials['params']) + if credentials['params']['SignatureVersion'] == '1': + return self._calc_signature_1(credentials['params']) + if credentials['params']['SignatureVersion'] == '2': + return self._calc_signature_2(credentials['params'], + credentials['verb'], + credentials['host'], + credentials['path']) raise Exception('Unknown Signature Version: %s' % - credentials.params['SignatureVersion']) + credentials['params']['SignatureVersion']) @staticmethod def _get_utf8_value(value): @@ -115,7 +115,7 @@ class Ec2Signer(object): def _calc_signature_2(self, params, verb, server_string, path): """Generate AWS signature version 2 string.""" - LOG.debug('using _calc_signature_2') + logging.debug('using _calc_signature_2') string_to_sign = '%s\n%s\n%s\n' % (verb, server_string, path) if self.hmac_256: current_hmac = self.hmac_256 @@ -131,13 +131,13 @@ class Ec2Signer(object): val = urllib.quote(val, safe='-_~') pairs.append(urllib.quote(key, safe='') + '=' + val) qs = '&'.join(pairs) - LOG.debug('query string: %s', qs) + logging.debug('query string: %s', qs) string_to_sign += qs - LOG.debug('string_to_sign: %s', string_to_sign) + logging.debug('string_to_sign: %s', string_to_sign) current_hmac.update(string_to_sign) b64 = base64.b64encode(current_hmac.digest()) - LOG.debug('len(b64)=%d', len(b64)) - LOG.debug('base64 encoded digest: %s', b64) + logging.debug('len(b64)=%d', len(b64)) + logging.debug('base64 encoded digest: %s', b64) return b64 diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 0d7725a164..ce424c6c9b 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -22,6 +22,8 @@ glance to list images needed to perform the requested task. import uuid +import webob.exc + from keystone import catalog from keystone import config from keystone import identity @@ -86,7 +88,7 @@ class Ec2Controller(wsgi.Application): super(Ec2Controller, self).__init__() def check_signature(self, creds_ref, credentials): - signer = utils.Signer(creds_ref['secret']) + signer = utils.Ec2Signer(creds_ref['secret']) signature = signer.generate(credentials) if signature == credentials['signature']: return @@ -98,9 +100,11 @@ class Ec2Controller(wsgi.Application): signature = signer.generate(credentials) if signature != credentials.signature: # TODO(termie): proper exception - raise Exception("Not Authorized") + msg = "Invalid signature" + raise webob.exc.HTTPUnauthorized(explanation=msg) else: - raise Exception("Not Authorized") + msg = "Signature not supplied" + raise webob.exc.HTTPUnauthorized(explanation=msg) def authenticate(self, context, credentials=None, ec2Credentials=None): @@ -129,8 +133,13 @@ class Ec2Controller(wsgi.Application): # NOTE(termie): backwards compat hack if not credentials and ec2Credentials: credentials = ec2Credentials + creds_ref = self.ec2_api.get_credential(context, credentials['access']) + if not creds_ref: + msg = "Access key not found" + raise webob.exc.HTTPUnauthorized(explanation=msg) + self.check_signature(creds_ref, credentials) From 9dadf0162448151aaa769f2ef5555e36616b4b7d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 6 Feb 2012 23:05:19 +0000 Subject: [PATCH 296/334] remove extra line --- keystone/contrib/ec2/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index ce424c6c9b..da493441af 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -140,7 +140,6 @@ class Ec2Controller(wsgi.Application): msg = "Access key not found" raise webob.exc.HTTPUnauthorized(explanation=msg) - self.check_signature(creds_ref, credentials) # TODO(termie): don't create new tokens every time From fa5b2e450a441fb6d838f842c46cd363775ba3d0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 6 Feb 2012 16:36:41 -0800 Subject: [PATCH 297/334] We don't need all the deps to check pep8. Change-Id: I01296b99d7ed91eb26b3d29256d8b9d538d9c839 --- tox.ini | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 96c4e0da9f..8779ba73f1 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ deps = -r{toxinidir}/tools/pip-requires-test commands = nosetests [testenv:pep8] +deps = pep8 commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin keystone setup.py [testenv:hudson] @@ -18,9 +19,3 @@ deps = file://{toxinidir}/.cache.bundle [testenv:jenkins27] basepython = python2.7 deps = file://{toxinidir}/.cache.bundle - -[testenv:jenkinspep8] -deps = file://{toxinidir}/.cache.bundle -commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin keystone setup.py - - From fabad5a6604e9e0248ec310e2b808dc06799c34b Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 6 Feb 2012 18:01:08 -0800 Subject: [PATCH 298/334] remove novaclient, fix python syntax Change-Id: Ib5c523a5feb74fbc6f4f75ec4d112dbcd23a559c --- tests/test_keystoneclient.py | 3 +- tests/test_novaclient_compat.py | 62 --------------------------------- 2 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 tests/test_novaclient_compat.py diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index a32a3b143c..81d4175263 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -136,8 +136,7 @@ class KeystoneClientTests(object): # FIXME(dolph): this should raise unauthorized # from keystoneclient import exceptions as client_exceptions # with self.assertRaises(client_exceptions.Unauthorized): - with self.assertRaises(Exception): - token_client.tenants.list() + self.assertRaises(Exception, token_client.tenants.list) # TODO(termie): I'm not really sure that this is testing much def test_endpoints(self): diff --git a/tests/test_novaclient_compat.py b/tests/test_novaclient_compat.py deleted file mode 100644 index 5578e4f784..0000000000 --- a/tests/test_novaclient_compat.py +++ /dev/null @@ -1,62 +0,0 @@ -import copy -import json -import os -import sys - -from keystone import config -from keystone import test -from keystone.common import logging -from keystone.common import utils - -import default_fixtures - - -CONF = config.CONF -OPENSTACK_REPO = 'https://review.openstack.org/p/openstack' -NOVACLIENT_REPO = '%s/python-novaclient.git' % OPENSTACK_REPO - - -class CompatTestCase(test.TestCase): - def setUp(self): - super(CompatTestCase, self).setUp() - - -class NovaClientCompatMasterTestCase(CompatTestCase): - def setUp(self): - super(NovaClientCompatMasterTestCase, self).setUp() - - revdir = test.checkout_vendor(NOVACLIENT_REPO, 'master') - self.add_path(revdir) - from novaclient.keystone import client as ks_client - from novaclient import client as base_client - reload(ks_client) - reload(base_client) - - CONF(config_files=[test.etcdir('keystone.conf'), - test.testsdir('test_overrides.conf')]) - self.app = self.loadapp('keystone') - self.load_backends() - self.load_fixtures(default_fixtures) - self.server = self.serveapp('keystone') - - def test_authenticate_and_tenants(self): - from novaclient.keystone import client as ks_client - from novaclient import client as base_client - - port = self.server.socket_info['socket'][1] - CONF.public_port = port - - # NOTE(termie): novaclient wants a "/" TypeErrorat the end, keystoneclient does not - # NOTE(termie): projectid is apparently sent as tenantName, so... that's - # unfortunate. - # NOTE(termie): novaclient seems to care about the region more than - # keystoneclient - conn = base_client.HTTPClient(auth_url="http://localhost:%s/v2.0/" % port, - user='FOO', - password='foo2', - projectid='BAR', - region_name='RegionOne') - client = ks_client.Client(conn) - client.authenticate() - # NOTE(termie): novaclient doesn't know about tenants or anything like that - # so just test that we can validate From aed78aa21a84881e6abaa662a5bb345db830e8e7 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 6 Feb 2012 20:06:31 -0800 Subject: [PATCH 299/334] fixes lp:925721 adds .gitreview for redux branch Change-Id: Ic993ee7abd5f54b118b1b58688199f58a529222c --- .gitreview | 4 ++++ docs/source/configuration.rst | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 .gitreview diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000000..cd914fe02a --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/keystone.git diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 8c998700d2..586b66bc8e 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -179,6 +179,18 @@ example:: description="those other guys" \ name=tog +``tenant get`` +^^^^^^^^^^^^^^ + +keyword arguments + +* tenant_id + +example:: + + keystone-manage tenant get \ + tenant_id=523df7c89ce34640996d3d804cbc56f4 + Users ----- From 3efce6da3a3560328103892bf4dba92cc4305594 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Feb 2012 22:37:38 -0800 Subject: [PATCH 300/334] make pip requires match nova Change-Id: Ie64d9571730f699d276fbf93241506e63053f0e0 --- tools/pip-requires | 4 ++-- tools/pip-requires-test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/pip-requires b/tools/pip-requires index 94ab44ba2c..03230ae854 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,7 +1,7 @@ # keystonelight dependencies pam==0.1.4 -WebOb==0.9.8 -eventlet==0.9.12 +WebOb==1.0.8 +eventlet PasteDeploy paste routes diff --git a/tools/pip-requires-test b/tools/pip-requires-test index e40795c826..46beec26ff 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -1,7 +1,7 @@ # keystonelight dependencies pam==0.1.4 -WebOb==0.9.8 -eventlet==0.9.12 +WebOb==1.0.8 +eventlet PasteDeploy paste routes From f0f8ddeaa8923af615fad9171c96ea1ef1f5968d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Feb 2012 23:07:10 -0800 Subject: [PATCH 301/334] Ensures duplicate users and tenants can't be made * adds test for duplicate names and ids for backends * also adds test for rename duplicates and changing ids * makes kvs backend raise an exception if duplicate is requested * ensures kvs backend doesn't allow update of id * makes sure that kvs is reset between tests * cleans up a few imports * fixes bug 927291 * fixes bug 928659 Change-Id: Ia6eb1961796cbde7ed57a75cd9394d77c88cf655 --- keystone/identity/backends/kvs.py | 23 ++- keystone/test.py | 5 +- tests/test_backend.py | 269 ++++++++++++++++++++---------- 3 files changed, 203 insertions(+), 94 deletions(-) diff --git a/keystone/identity/backends/kvs.py b/keystone/identity/backends/kvs.py index 8215054b49..7dfc863370 100644 --- a/keystone/identity/backends/kvs.py +++ b/keystone/identity/backends/kvs.py @@ -125,6 +125,10 @@ class Identity(kvs.Base, identity.Driver): # CRUD def create_user(self, user_id, user): + if self.get_user(user_id): + raise Exception('Duplicate id') + if self.get_user_by_name(user['name']): + raise Exception('Duplicate name') user = _ensure_hashed_password(user) self.db.set('user-%s' % user_id, user) self.db.set('user_name-%s' % user['name'], user) @@ -134,11 +138,16 @@ class Identity(kvs.Base, identity.Driver): return user def update_user(self, user_id, user): + if 'name' in user: + existing = self.db.get('user_name-%s' % user['name']) + if existing and user_id != existing['id']: + raise Exception('Duplicate name') # get the old name and delete it too old_user = self.db.get('user-%s' % user_id) new_user = old_user.copy() user = _ensure_hashed_password(user) new_user.update(user) + new_user['id'] = user_id self.db.delete('user_name-%s' % old_user['name']) self.db.set('user-%s' % user_id, new_user) self.db.set('user_name-%s' % new_user['name'], new_user) @@ -154,16 +163,26 @@ class Identity(kvs.Base, identity.Driver): return None def create_tenant(self, tenant_id, tenant): + if self.get_tenant(tenant_id): + raise Exception('Duplicate id') + if self.get_tenant_by_name(tenant['name']): + raise Exception('Duplicate name') self.db.set('tenant-%s' % tenant_id, tenant) self.db.set('tenant_name-%s' % tenant['name'], tenant) return tenant def update_tenant(self, tenant_id, tenant): + if 'name' in tenant: + existing = self.db.get('tenant_name-%s' % tenant['name']) + if existing and tenant_id != existing['id']: + raise Exception('Duplicate name') # get the old name and delete it too old_tenant = self.db.get('tenant-%s' % tenant_id) + new_tenant = old_tenant.copy() + new_tenant['id'] = tenant_id self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % tenant_id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) + self.db.set('tenant-%s' % tenant_id, new_tenant) + self.db.set('tenant_name-%s' % new_tenant['name'], new_tenant) return tenant def delete_tenant(self, tenant_id): diff --git a/keystone/test.py b/keystone/test.py index 720b4db832..1eeff449ab 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -8,10 +8,8 @@ import time from paste import deploy -from keystone import catalog from keystone import config -from keystone import identity -from keystone import token +from keystone.common import kvs from keystone.common import logging from keystone.common import utils from keystone.common import wsgi @@ -119,6 +117,7 @@ class TestCase(unittest.TestCase): for path in self._paths: if path in sys.path: sys.path.remove(path) + kvs.INMEMDB.clear() CONF.reset() super(TestCase, self).tearDown() diff --git a/tests/test_backend.py b/tests/test_backend.py index 151700aa23..b106f23f2c 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,108 +1,199 @@ class IdentityTests(object): - def test_authenticate_bad_user(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) - def test_authenticate_bad_password(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password'] + 'WRONG') + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') - def test_authenticate_invalid_tenant(self): - self.assertRaises(AssertionError, - self.identity_api.authenticate, - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG', - password=self.user_foo['password']) + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) - def test_authenticate_no_tenant(self): - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - password=self.user_foo['password']) - # NOTE(termie): the password field is left in user_foo to make it easier - # to authenticate in tests, but should not be returned by - # the api - self.user_foo.pop('password') - self.assertDictEquals(user_ref, self.user_foo) - self.assert_(tenant_ref is None) - self.assert_(not metadata_ref) + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(not metadata_ref) - def test_authenticate(self): - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'], - password=self.user_foo['password']) - # NOTE(termie): the password field is left in user_foo to make it easier - # to authenticate in tests, but should not be returned by - # the api - self.user_foo.pop('password') - self.assertDictEquals(user_ref, self.user_foo) - self.assertDictEquals(tenant_ref, self.tenant_bar) - self.assertDictEquals(metadata_ref, self.metadata_foobar) + def test_authenticate(self): + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(metadata_ref, self.metadata_foobar) - def test_password_hashed(self): - user_ref = self.identity_api._get_user(self.user_foo['id']) - self.assertNotEqual(user_ref['password'], self.user_foo['password']) + def test_password_hashed(self): + user_ref = self.identity_api._get_user(self.user_foo['id']) + self.assertNotEqual(user_ref['password'], self.user_foo['password']) - def test_get_tenant_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(tenant_ref is None) + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) - def test_get_tenant(self): - tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) - self.assertDictEquals(tenant_ref, self.tenant_bar) + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) - def test_get_tenant_by_name_bad_tenant(self): - tenant_ref = self.identity_api.get_tenant( - tenant_id=self.tenant_bar['name'] + 'WRONG') - self.assert_(tenant_ref is None) + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) - def test_get_tenant_by_name(self): - tenant_ref = self.identity_api.get_tenant_by_name( - tenant_name=self.tenant_bar['name']) - self.assertDictEquals(tenant_ref, self.tenant_bar) + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) - def test_get_user_bad_user(self): - user_ref = self.identity_api.get_user( - user_id=self.user_foo['id'] + 'WRONG') - self.assert_(user_ref is None) + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) - def test_get_user(self): - user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) - # NOTE(termie): the password field is left in user_foo to make it easier - # to authenticate in tests, but should not be returned by - # the api - self.user_foo.pop('password') - self.assertDictEquals(user_ref, self.user_foo) + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + # NOTE(termie): the password field is left in user_foo to make it easier + # to authenticate in tests, but should not be returned by + # the api + self.user_foo.pop('password') + self.assertDictEquals(user_ref, self.user_foo) - def test_get_metadata_bad_user(self): - metadata_ref = self.identity_api.get_metadata( - user_id=self.user_foo['id'] + 'WRONG', - tenant_id=self.tenant_bar['id']) - self.assert_(metadata_ref is None) + def test_get_metadata_bad_user(self): + metadata_ref = self.identity_api.get_metadata( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(metadata_ref is None) - def test_get_metadata_bad_tenant(self): - metadata_ref = self.identity_api.get_metadata( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id'] + 'WRONG') - self.assert_(metadata_ref is None) + def test_get_metadata_bad_tenant(self): + metadata_ref = self.identity_api.get_metadata( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(metadata_ref is None) - def test_get_metadata(self): - metadata_ref = self.identity_api.get_metadata( - user_id=self.user_foo['id'], - tenant_id=self.tenant_bar['id']) - self.assertDictEquals(metadata_ref, self.metadata_foobar) + def test_get_metadata(self): + metadata_ref = self.identity_api.get_metadata( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(metadata_ref, self.metadata_foobar) - def test_get_role(self): - role_ref = self.identity_api.get_role( - role_id=self.role_keystone_admin['id']) - self.assertDictEquals(role_ref, self.role_keystone_admin) + def test_get_role(self): + role_ref = self.identity_api.get_role( + role_id=self.role_keystone_admin['id']) + self.assertDictEquals(role_ref, self.role_keystone_admin) + def test_create_duplicate_user_id_fails(self): + user = {'id': 'fake1', + 'name': 'fake1', + 'password': 'fakepass', + 'tenants': ['bar',]} + self.identity_api.create_user('fake1', user) + user['name'] = 'fake2' + self.assertRaises(Exception, + self.identity_api.create_user, + 'fake1', + user) + def test_create_duplicate_user_name_fails(self): + user = {'id': 'fake1', + 'name': 'fake1', + 'password': 'fakepass', + 'tenants': ['bar',]} + self.identity_api.create_user('fake1', user) + user['id'] = 'fake2' + self.assertRaises(Exception, + self.identity_api.create_user, + 'fake2', + user) + + def test_rename_duplicate_user_name_fails(self): + user1 = {'id': 'fake1', + 'name': 'fake1', + 'password': 'fakepass', + 'tenants': ['bar',]} + user2 = {'id': 'fake2', + 'name': 'fake2', + 'password': 'fakepass', + 'tenants': ['bar',]} + self.identity_api.create_user('fake1', user1) + self.identity_api.create_user('fake2', user2) + user2['name'] = 'fake1' + self.assertRaises(Exception, + self.identity_api.update_user, + 'fake2', + user2) + + def test_update_user_id_does_nothing(self): + user = {'id': 'fake1', + 'name': 'fake1', + 'password': 'fakepass', + 'tenants': ['bar',]} + self.identity_api.create_user('fake1', user) + user['id'] = 'fake2' + self.identity_api.update_user('fake1', user) + user_ref = self.identity_api.get_user('fake1') + self.assertEqual(user_ref['id'], 'fake1') + user_ref = self.identity_api.get_user('fake2') + self.assert_(user_ref is None) + + def test_create_duplicate_tenant_id_fails(self): + tenant = {'id': 'fake1', 'name': 'fake1'} + self.identity_api.create_tenant('fake1', tenant) + tenant['name'] = 'fake2' + self.assertRaises(Exception, + self.identity_api.create_tenant, + 'fake1', + tenant) + + def test_create_duplicate_tenant_name_fails(self): + tenant = {'id': 'fake1', 'name': 'fake'} + self.identity_api.create_tenant('fake1', tenant) + tenant['id'] = 'fake2' + self.assertRaises(Exception, + self.identity_api.create_tenant, + 'fake1', + tenant) + + def test_rename_duplicate_tenant_name_fails(self): + tenant1 = {'id': 'fake1', 'name': 'fake1'} + tenant2 = {'id': 'fake2', 'name': 'fake2'} + self.identity_api.create_tenant('fake1', tenant1) + self.identity_api.create_tenant('fake2', tenant2) + tenant2['name'] = 'fake1' + self.assertRaises(Exception, + self.identity_api.update_tenant, + 'fake2', + tenant2) + + def test_update_tenant_id_does_nothing(self): + tenant = {'id': 'fake1', 'name': 'fake1'} + self.identity_api.create_tenant('fake1', tenant) + tenant['id'] = 'fake2' + self.identity_api.update_tenant('fake1', tenant) + tenant_ref = self.identity_api.get_tenant('fake1') + self.assertEqual(tenant_ref['id'], 'fake1') + tenant_ref = self.identity_api.get_tenant('fake2') + self.assert_(tenant_ref is None) From f9a88277502b10416488d799d99f831299ad474e Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 8 Feb 2012 12:20:31 -0800 Subject: [PATCH 302/334] example in hacking was incorrect Change-Id: I0a345e72c8974c26fb911f2a676c062280ac9557 --- HACKING.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HACKING.rst b/HACKING.rst index a0d052a807..e9b36c64be 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -82,6 +82,8 @@ Example:: after the closing quotations. For more in-depth explanations for these decisions see http://www.python.org/dev/peps/pep-0257/ + A docstring ends with an empty line before the closing quotations. + Describe parameters and return values, using the Sphinx format; the appropriate syntax is as follows. @@ -91,6 +93,7 @@ Example:: :returns: return_type -- description of the return value :returns: description of the return value :raises: AttributeError, KeyError + """ From 51eda0155f987f098b941d1eed4cd21e03a9c4d2 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Wed, 8 Feb 2012 14:01:03 -0600 Subject: [PATCH 303/334] termie all the things Change-Id: Ib7b5fab2a09de8a9dcad8d8b0cf71c529e944f8c --- keystone/cli.py | 2 +- keystone/common/bufferedhttp.py | 4 +- keystone/common/cfg.py | 14 +-- keystone/common/sql/core.py | 8 +- keystone/common/sql/migration.py | 4 +- keystone/common/utils.py | 8 +- keystone/config.py | 6 +- keystone/contrib/admin_crud/core.py | 174 ++++++++++++++-------------- keystone/contrib/ec2/core.py | 8 +- keystone/contrib/s3/core.py | 2 +- keystone/middleware/auth_token.py | 76 ++++++------ keystone/middleware/core.py | 10 +- keystone/middleware/ec2_token.py | 4 +- keystone/middleware/swift_auth.py | 14 +-- keystone/test.py | 4 +- 15 files changed, 169 insertions(+), 169 deletions(-) diff --git a/keystone/cli.py b/keystone/cli.py index 2caab52bc8..9616f955db 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -317,7 +317,7 @@ class DictWrapper(dict): def print_commands(cmds): print - print "Available commands:" + print 'Available commands:' o = [] max_length = max([len(k) for k in cmds]) + 2 for k, cmd in sorted(cmds.iteritems()): diff --git a/keystone/common/bufferedhttp.py b/keystone/common/bufferedhttp.py index 769a9b8bf3..95caa4f400 100644 --- a/keystone/common/bufferedhttp.py +++ b/keystone/common/bufferedhttp.py @@ -95,8 +95,8 @@ class BufferedHTTPConnection(HTTPConnection): def getresponse(self): response = HTTPConnection.getresponse(self) - logging.debug(("HTTP PERF: %(time).5f seconds to %(method)s " - "%(host)s:%(port)s %(path)s)"), + logging.debug(('HTTP PERF: %(time).5f seconds to %(method)s ' + '%(host)s:%(port)s %(path)s)'), {'time': time.time() - self._connected_time, 'method': self._method, 'host': self.host, 'port': self.port, 'path': self._path}) return response diff --git a/keystone/common/cfg.py b/keystone/common/cfg.py index 9551ff7967..91c9546c10 100644 --- a/keystone/common/cfg.py +++ b/keystone/common/cfg.py @@ -223,9 +223,9 @@ class ArgsAlreadyParsedError(Error): """Raised if a CLI opt is registered after parsing.""" def __str__(self): - ret = "arguments already parsed" + ret = 'arguments already parsed' if self.msg: - ret += ": " + self.msg + ret += ': ' + self.msg return ret @@ -238,9 +238,9 @@ class NoSuchOptError(Error): def __str__(self): if self.group is None: - return "no such option: %s" % self.opt_name + return 'no such option: %s' % self.opt_name else: - return "no such option in group %s: %s" % (self.group.name, + return 'no such option in group %s: %s' % (self.group.name, self.opt_name) @@ -251,7 +251,7 @@ class NoSuchGroupError(Error): self.group_name = group_name def __str__(self): - return "no such group: %s" % self.group_name + return 'no such group: %s' % self.group_name class DuplicateOptError(Error): @@ -261,14 +261,14 @@ class DuplicateOptError(Error): self.opt_name = opt_name def __str__(self): - return "duplicate option: %s" % self.opt_name + return 'duplicate option: %s' % self.opt_name class TemplateSubstitutionError(Error): """Raised if an error occurs substituting a variable in an opt value.""" def __str__(self): - return "template substitution error: %s" % self.msg + return 'template substitution error: %s' % self.msg class ConfigFilesNotFoundError(Error): diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index 172a2a6653..2bd24f4835 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -102,12 +102,12 @@ class Base(object): """Return a SQLAlchemy engine.""" connection_dict = sqlalchemy.engine.url.make_url(CONF.sql.connection) - engine_args = {"pool_recycle": CONF.sql.idle_timeout, - "echo": False, + engine_args = {'pool_recycle': CONF.sql.idle_timeout, + 'echo': False, } - if "sqlite" in connection_dict.drivername: - engine_args["poolclass"] = sqlalchemy.pool.NullPool + if 'sqlite' in connection_dict.drivername: + engine_args['poolclass'] = sqlalchemy.pool.NullPool return sql.create_engine(CONF.sql.connection, **engine_args) diff --git a/keystone/common/sql/migration.py b/keystone/common/sql/migration.py index 11d459a1ae..0ea9cd12d3 100644 --- a/keystone/common/sql/migration.py +++ b/keystone/common/sql/migration.py @@ -36,7 +36,7 @@ except ImportError: # See LP Bug #717467 from migrate import exceptions as versioning_exceptions except ImportError: - sys.exit("python-migrate is not installed. Exiting.") + sys.exit('python-migrate is not installed. Exiting.') def db_sync(version=None): @@ -44,7 +44,7 @@ def db_sync(version=None): try: version = int(version) except ValueError: - raise Exception("version should be an integer") + raise Exception('version should be an integer') current_version = db_version() repo_path = _find_migrate_repo() diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 16692bd635..70ef590049 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -169,14 +169,14 @@ def check_output(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - >>> check_output(["ls", "-l", "/dev/null"]) + >>> check_output(['ls', '-l', '/dev/null']) 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], + >>> check_output(['/bin/sh', '-c', + ... 'ls -l non_existent_file ; exit 0'], ... stderr=STDOUT) 'ls: non_existent_file: No such file or directory\n' """ @@ -187,7 +187,7 @@ def check_output(*popenargs, **kwargs): output, unused_err = process.communicate() retcode = process.poll() if retcode: - cmd = kwargs.get("args") + cmd = kwargs.get('args') if cmd is None: cmd = popenargs[0] raise subprocess.CalledProcessError(retcode, cmd) diff --git a/keystone/config.py b/keystone/config.py index 09c18ca6ac..e95efe425b 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -57,8 +57,8 @@ def setup_logging(conf): logging.config.fileConfig(conf.log_config) return else: - raise RuntimeError("Unable to locate specified logging " - "config file: %s" % conf.log_config) + raise RuntimeError('Unable to locate specified logging ' + 'config file: %s' % conf.log_config) root_logger = logging.root if conf.debug: @@ -75,7 +75,7 @@ def setup_logging(conf): facility = getattr(logging.SysLogHandler, conf.syslog_log_facility) except AttributeError: - raise ValueError(_("Invalid syslog facility")) + raise ValueError(_('Invalid syslog facility')) handler = logging.SysLogHandler(address='/dev/log', facility=facility) diff --git a/keystone/contrib/admin_crud/core.py b/keystone/contrib/admin_crud/core.py index ce0867e711..15a597ed18 100644 --- a/keystone/contrib/admin_crud/core.py +++ b/keystone/contrib/admin_crud/core.py @@ -19,132 +19,132 @@ class CrudExtension(wsgi.ExtensionRouter): service_controller = catalog.ServiceController() # Tenant Operations - mapper.connect("/tenants", controller=tenant_controller, - action="create_tenant", - conditions=dict(method=["POST"])) - mapper.connect("/tenants/{tenant_id}", + mapper.connect('/tenants', controller=tenant_controller, + action='create_tenant', + conditions=dict(method=['POST'])) + mapper.connect('/tenants/{tenant_id}', controller=tenant_controller, - action="update_tenant", - conditions=dict(method=["PUT", "POST"])) - mapper.connect("/tenants/{tenant_id}", + action='update_tenant', + conditions=dict(method=['PUT', 'POST'])) + mapper.connect('/tenants/{tenant_id}', controller=tenant_controller, - action="delete_tenant", - conditions=dict(method=["DELETE"])) - mapper.connect("/tenants/{tenant_id}/users", + action='delete_tenant', + conditions=dict(method=['DELETE'])) + mapper.connect('/tenants/{tenant_id}/users', controller=user_controller, - action="get_tenant_users", - conditions=dict(method=["GET"])) + action='get_tenant_users', + conditions=dict(method=['GET'])) # User Operations - mapper.connect("/users", + mapper.connect('/users', controller=user_controller, - action="get_users", - conditions=dict(method=["GET"])) - mapper.connect("/users", + action='get_users', + conditions=dict(method=['GET'])) + mapper.connect('/users', controller=user_controller, - action="create_user", - conditions=dict(method=["POST"])) + action='create_user', + conditions=dict(method=['POST'])) # NOTE(termie): not in diablo - mapper.connect("/users/{user_id}", + mapper.connect('/users/{user_id}', controller=user_controller, - action="update_user", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}", + action='update_user', + conditions=dict(method=['PUT'])) + mapper.connect('/users/{user_id}', controller=user_controller, - action="delete_user", - conditions=dict(method=["DELETE"])) + action='delete_user', + conditions=dict(method=['DELETE'])) # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/password", + mapper.connect('/users/{user_id}/password', controller=user_controller, - action="set_user_password", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/password", + action='set_user_password', + conditions=dict(method=['PUT'])) + mapper.connect('/users/{user_id}/OS-KSADM/password', controller=user_controller, - action="set_user_password", - conditions=dict(method=["PUT"])) + action='set_user_password', + conditions=dict(method=['PUT'])) # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/tenant", + mapper.connect('/users/{user_id}/tenant', controller=user_controller, - action="update_user_tenant", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/tenant", + action='update_user_tenant', + conditions=dict(method=['PUT'])) + mapper.connect('/users/{user_id}/OS-KSADM/tenant', controller=user_controller, - action="update_user_tenant", - conditions=dict(method=["PUT"])) + action='update_user_tenant', + conditions=dict(method=['PUT'])) # COMPAT(diablo): the copy with no OS-KSADM is from diablo - mapper.connect("/users/{user_id}/enabled", + mapper.connect('/users/{user_id}/enabled', controller=user_controller, - action="set_user_enabled", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/OS-KSADM/enabled", + action='set_user_enabled', + conditions=dict(method=['PUT'])) + mapper.connect('/users/{user_id}/OS-KSADM/enabled', controller=user_controller, - action="set_user_enabled", - conditions=dict(method=["PUT"])) + action='set_user_enabled', + conditions=dict(method=['PUT'])) # User Roles - mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="add_role_to_user", - conditions=dict(method=["PUT"])) - mapper.connect("/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="delete_role_from_user", - conditions=dict(method=["DELETE"])) + mapper.connect('/users/{user_id}/roles/OS-KSADM/{role_id}', + controller=role_controller, action='add_role_to_user', + conditions=dict(method=['PUT'])) + mapper.connect('/users/{user_id}/roles/OS-KSADM/{role_id}', + controller=role_controller, action='delete_role_from_user', + conditions=dict(method=['DELETE'])) # COMPAT(diablo): User Roles - mapper.connect("/users/{user_id}/roleRefs", - controller=role_controller, action="get_role_refs", - conditions=dict(method=["GET"])) - mapper.connect("/users/{user_id}/roleRefs", - controller=role_controller, action="create_role_ref", - conditions=dict(method=["POST"])) - mapper.connect("/users/{user_id}/roleRefs/{role_ref_id}", - controller=role_controller, action="delete_role_ref", - conditions=dict(method=["DELETE"])) + mapper.connect('/users/{user_id}/roleRefs', + controller=role_controller, action='get_role_refs', + conditions=dict(method=['GET'])) + mapper.connect('/users/{user_id}/roleRefs', + controller=role_controller, action='create_role_ref', + conditions=dict(method=['POST'])) + mapper.connect('/users/{user_id}/roleRefs/{role_ref_id}', + controller=role_controller, action='delete_role_ref', + conditions=dict(method=['DELETE'])) # User-Tenant Roles mapper.connect( - "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="add_role_to_user", - conditions=dict(method=["PUT"])) + '/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}', + controller=role_controller, action='add_role_to_user', + conditions=dict(method=['PUT'])) mapper.connect( - "/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}", - controller=role_controller, action="remove_role_from_user", - conditions=dict(method=["DELETE"])) + '/tenants/{tenant_id}/users/{user_id}/roles/OS-KSADM/{role_id}', + controller=role_controller, action='remove_role_from_user', + conditions=dict(method=['DELETE'])) # Service Operations - mapper.connect("/OS-KSADM/services", + mapper.connect('/OS-KSADM/services', controller=service_controller, - action="get_services", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/services", + action='get_services', + conditions=dict(method=['GET'])) + mapper.connect('/OS-KSADM/services', controller=service_controller, - action="create_service", - conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/services/{service_id}", + action='create_service', + conditions=dict(method=['POST'])) + mapper.connect('/OS-KSADM/services/{service_id}', controller=service_controller, - action="delete_service", - conditions=dict(method=["DELETE"])) - mapper.connect("/OS-KSADM/services/{service_id}", + action='delete_service', + conditions=dict(method=['DELETE'])) + mapper.connect('/OS-KSADM/services/{service_id}', controller=service_controller, - action="get_service", - conditions=dict(method=["GET"])) + action='get_service', + conditions=dict(method=['GET'])) # Role Operations - mapper.connect("/OS-KSADM/roles", + mapper.connect('/OS-KSADM/roles', controller=role_controller, - action="create_role", - conditions=dict(method=["POST"])) - mapper.connect("/OS-KSADM/roles", + action='create_role', + conditions=dict(method=['POST'])) + mapper.connect('/OS-KSADM/roles', controller=role_controller, - action="get_roles", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/roles/{role_id}", + action='get_roles', + conditions=dict(method=['GET'])) + mapper.connect('/OS-KSADM/roles/{role_id}', controller=role_controller, - action="get_role", - conditions=dict(method=["GET"])) - mapper.connect("/OS-KSADM/roles/{role_id}", + action='get_role', + conditions=dict(method=['GET'])) + mapper.connect('/OS-KSADM/roles/{role_id}', controller=role_controller, - action="delete_role", - conditions=dict(method=["DELETE"])) + action='delete_role', + conditions=dict(method=['DELETE'])) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index da493441af..851bc588c7 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -95,15 +95,15 @@ class Ec2Controller(wsgi.Application): # NOTE(vish): Some libraries don't use the port when signing # requests, so try again without port. elif ':' in credentials['signature']: - hostname, _port = credentials['host'].split(":") + hostname, _port = credentials['host'].split(':') credentials['host'] = hostname signature = signer.generate(credentials) if signature != credentials.signature: # TODO(termie): proper exception - msg = "Invalid signature" + msg = 'Invalid signature' raise webob.exc.HTTPUnauthorized(explanation=msg) else: - msg = "Signature not supplied" + msg = 'Signature not supplied' raise webob.exc.HTTPUnauthorized(explanation=msg) def authenticate(self, context, credentials=None, @@ -137,7 +137,7 @@ class Ec2Controller(wsgi.Application): creds_ref = self.ec2_api.get_credential(context, credentials['access']) if not creds_ref: - msg = "Access key not found" + msg = 'Access key not found' raise webob.exc.HTTPUnauthorized(explanation=msg) self.check_signature(creds_ref, credentials) diff --git a/keystone/contrib/s3/core.py b/keystone/contrib/s3/core.py index 208c6aa555..d69fade4f1 100644 --- a/keystone/contrib/s3/core.py +++ b/keystone/contrib/s3/core.py @@ -34,4 +34,4 @@ class S3Controller(ec2.Ec2Controller): signed = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() if credentials['signature'] != signed: - raise Exception("Not Authorized") + raise Exception('Not Authorized') diff --git a/keystone/middleware/auth_token.py b/keystone/middleware/auth_token.py index d806dd01f7..0dc723b9c6 100755 --- a/keystone/middleware/auth_token.py +++ b/keystone/middleware/auth_token.py @@ -78,7 +78,7 @@ from webob.exc import HTTPUnauthorized from keystone.common.bufferedhttp import http_connect_raw as http_connect -PROTOCOL_NAME = "Token Authentication" +PROTOCOL_NAME = 'Token Authentication' class AuthProtocol(object): @@ -86,7 +86,7 @@ class AuthProtocol(object): def _init_protocol_common(self, app, conf): """ Common initialization code""" - print "Starting the %s component" % PROTOCOL_NAME + print 'Starting the %s component' % PROTOCOL_NAME self.conf = conf self.app = app @@ -120,7 +120,7 @@ class AuthProtocol(object): # where to tell clients to find the auth service (default to url # constructed based on endpoint we have for the service to use) self.auth_location = conf.get('auth_uri', - "%s://%s:%s" % (self.auth_protocol, + '%s://%s:%s' % (self.auth_protocol, self.auth_host, self.auth_port)) @@ -152,8 +152,8 @@ class AuthProtocol(object): if self.delay_auth_decision: #Configured to allow downstream service to make final decision. #So mark status as Invalid and forward the request downstream - self._decorate_request("X_IDENTITY_STATUS", - "Invalid", env, proxy_headers) + self._decorate_request('X_IDENTITY_STATUS', + 'Invalid', env, proxy_headers) else: #Respond to client as appropriate for this auth protocol return self._reject_request(env, start_response) @@ -164,14 +164,14 @@ class AuthProtocol(object): # Keystone rejected claim if self.delay_auth_decision: # Downstream service will receive call still and decide - self._decorate_request("X_IDENTITY_STATUS", - "Invalid", env, proxy_headers) + self._decorate_request('X_IDENTITY_STATUS', + 'Invalid', env, proxy_headers) else: #Respond to client as appropriate for this auth protocol return self._reject_claims(env, start_response) else: - self._decorate_request("X_IDENTITY_STATUS", - "Confirmed", env, proxy_headers) + self._decorate_request('X_IDENTITY_STATUS', + 'Confirmed', env, proxy_headers) #Collect information about valid claims if valid: @@ -179,7 +179,7 @@ class AuthProtocol(object): # Store authentication data if claims: - self._decorate_request('X_AUTHORIZATION', "Proxy %s" % + self._decorate_request('X_AUTHORIZATION', 'Proxy %s' % claims['user'], env, proxy_headers) # For legacy compatibility before we had ID and Name @@ -218,14 +218,14 @@ class AuthProtocol(object): validate a user's token. Validate_token is a priviledged call so it needs to be authenticated by a service that is calling it """ - headers = {"Content-type": "application/json", - "Accept": "application/json"} - params = {"passwordCredentials": {"username": username, - "password": password, - "tenantId": "1"}} - conn = httplib.HTTPConnection("%s:%s" \ + headers = {'Content-type': 'application/json', + 'Accept': 'application/json'} + params = {'passwordCredentials': {'username': username, + 'password': password, + 'tenantId': '1'}} + conn = httplib.HTTPConnection('%s:%s' \ % (self.auth_host, self.auth_port)) - conn.request("POST", "/v2.0/tokens", json.dumps(params), \ + conn.request('POST', '/v2.0/tokens', json.dumps(params), \ headers=headers) response = conn.getresponse() data = response.read() @@ -238,8 +238,8 @@ class AuthProtocol(object): def _reject_request(self, env, start_response): """Redirect client to auth server""" - return webob.exc.HTTPUnauthorized("Authentication required", - [("WWW-Authenticate", + return webob.exc.HTTPUnauthorized('Authentication required', + [('WWW-Authenticate', "Keystone uri='%s'" % self.auth_location)])(env, start_response) @@ -255,19 +255,19 @@ class AuthProtocol(object): # admin token #TODO(ziad): Need to properly implement this, where to store creds # for now using token from ini - #auth = self.get_admin_auth_token("admin", "secrete", "1") - #admin_token = json.loads(auth)["auth"]["token"]["id"] + #auth = self.get_admin_auth_token('admin', 'secrete', '1') + #admin_token = json.loads(auth)['auth']['token']['id'] # Step 2: validate the user's token with the auth service # since this is a priviledged op,m we need to auth ourselves # by using an admin token - headers = {"Content-type": "application/json", - "Accept": "application/json", - "X-Auth-Token": self.admin_token} + headers = {'Content-type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': self.admin_token} ##TODO(ziad):we need to figure out how to auth to keystone #since validate_token is a priviledged call #Khaled's version uses creds to get a token - # "X-Auth-Token": admin_token} + # 'X-Auth-Token': admin_token} # we're using a test token from the ini file for now conn = http_connect(self.auth_host, self.auth_port, 'GET', '/v2.0/tokens/%s' % claims, headers=headers) @@ -287,13 +287,13 @@ class AuthProtocol(object): def _expound_claims(self, claims): # Valid token. Get user data and put it in to the call # so the downstream service can use it - headers = {"Content-type": "application/json", - "Accept": "application/json", - "X-Auth-Token": self.admin_token} + headers = {'Content-type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': self.admin_token} ##TODO(ziad):we need to figure out how to auth to keystone #since validate_token is a priviledged call #Khaled's version uses creds to get a token - # "X-Auth-Token": admin_token} + # 'X-Auth-Token': admin_token} # we're using a test token from the ini file for now conn = http_connect(self.auth_host, self.auth_port, 'GET', '/v2.0/tokens/%s' % claims, headers=headers) @@ -306,12 +306,12 @@ class AuthProtocol(object): token_info = json.loads(data) roles = [] - role_refs = token_info["access"]["user"]["roles"] + role_refs = token_info['access']['user']['roles'] if role_refs != None: for role_ref in role_refs: # Nova looks for the non case-sensitive role 'Admin' # to determine admin-ness - roles.append(role_ref["name"]) + roles.append(role_ref['name']) try: tenant = token_info['access']['token']['tenant']['id'] @@ -332,12 +332,12 @@ class AuthProtocol(object): def _decorate_request(self, index, value, env, proxy_headers): """Add headers to request""" proxy_headers[index] = value - env["HTTP_%s" % index] = value + env['HTTP_%s' % index] = value def _forward_request(self, env, start_response, proxy_headers): """Token/Auth processed & claims added to headers""" self._decorate_request('AUTHORIZATION', - "Basic %s" % self.service_pass, env, proxy_headers) + 'Basic %s' % self.service_pass, env, proxy_headers) #now decide how to pass on the call if self.app: # Pass to downstream WSGI component @@ -362,7 +362,7 @@ class AuthProtocol(object): if resp.status == 401 or resp.status == 305: # Add our own headers to the list - headers = [("WWW_AUTHENTICATE", + headers = [('WWW_AUTHENTICATE', "Keystone uri='%s'" % self.auth_location)] return webob.Response(status=resp.status, body=data, @@ -387,11 +387,11 @@ def app_factory(global_conf, **local_conf): conf.update(local_conf) return AuthProtocol(None, conf) -if __name__ == "__main__": - app = deploy.loadapp("config:" + \ +if __name__ == '__main__': + app = deploy.loadapp('config:' + \ os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir, - "examples/paste/auth_token.ini"), - global_conf={"log_name": "auth_token.log"}) + 'examples/paste/auth_token.ini'), + global_conf={'log_name': 'auth_token.log'}) wsgi.server(eventlet.listen(('', 8090)), app) diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 46d38673b8..8e206ddef4 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -115,15 +115,15 @@ class Debug(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): - print ("*" * 40) + " REQUEST ENVIRON" + print ('*' * 40) + ' REQUEST ENVIRON' for key, value in req.environ.items(): - print key, "=", value + print key, '=', value print resp = req.get_response(self.application) - print ("*" * 40) + " RESPONSE HEADERS" + print ('*' * 40) + ' RESPONSE HEADERS' for (key, value) in resp.headers.iteritems(): - print key, "=", value + print key, '=', value print resp.app_iter = self.print_generator(resp.app_iter) @@ -136,7 +136,7 @@ class Debug(wsgi.Middleware): Iterator that prints the contents of a wrapper string iterator when iterated. """ - print ("*" * 40) + " BODY" + print ('*' * 40) + ' BODY' for part in app_iter: sys.stdout.write(part) sys.stdout.flush() diff --git a/keystone/middleware/ec2_token.py b/keystone/middleware/ec2_token.py index af1416273f..cc3094a3be 100644 --- a/keystone/middleware/ec2_token.py +++ b/keystone/middleware/ec2_token.py @@ -65,11 +65,11 @@ class EC2Token(wsgi.Middleware): creds_json = utils.dumps(creds) headers = {'Content-Type': 'application/json'} - # Disable "has no x member" pylint error + # Disable 'has no x member' pylint error # for httplib and urlparse # pylint: disable-msg=E1101 o = urlparse(FLAGS.keystone_ec2_url) - if o.scheme == "http": + if o.scheme == 'http': conn = httplib.HTTPConnection(o.netloc) else: conn = httplib.HTTPSConnection(o.netloc) diff --git a/keystone/middleware/swift_auth.py b/keystone/middleware/swift_auth.py index 35bab6a337..e83c13667d 100755 --- a/keystone/middleware/swift_auth.py +++ b/keystone/middleware/swift_auth.py @@ -49,7 +49,7 @@ from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed from swift.common.utils import get_logger, split_path -PROTOCOL_NAME = "Swift Token Authentication" +PROTOCOL_NAME = 'Swift Token Authentication' class AuthProtocol(object): @@ -195,9 +195,9 @@ class AuthProtocol(object): # TODO(todd): cache self.log.debug('Asking keystone to validate token') - headers = {"Content-type": "application/json", - "Accept": "application/json", - "X-Auth-Token": self.admin_token} + headers = {'Content-type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': self.admin_token} self.log.debug('headers: %r', headers) self.log.debug('url: %s', self.keystone_url) conn = http_connect(self.keystone_url.hostname, self.keystone_url.port, @@ -206,17 +206,17 @@ class AuthProtocol(object): data = resp.read() conn.close() - # Check http status code for the "OK" family of responses + # Check http status code for the 'OK' family of responses if not str(resp.status).startswith('20'): return False identity_info = json.loads(data) roles = [] - role_refs = identity_info["access"]["user"]["roles"] + role_refs = identity_info['access']['user']['roles'] if role_refs is not None: for role_ref in role_refs: - roles.append(role_ref["id"]) + roles.append(role_ref['id']) try: tenant = identity_info['access']['token']['tenantId'] diff --git a/keystone/test.py b/keystone/test.py index 1eeff449ab..f28ce6f709 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -227,12 +227,12 @@ class TestCase(unittest.TestCase): def assertDictEquals(self, actual, expected): for k in expected: self.assertTrue(k in actual, - "Expected key %s not in %s." % (k, actual)) + 'Expected key %s not in %s.' % (k, actual)) self.assertDeepEquals(expected[k], actual[k]) for k in actual: self.assertTrue(k in expected, - "Unexpected key %s in %s." % (k, actual)) + 'Unexpected key %s in %s.' % (k, actual)) def assertDeepEquals(self, actual, expected): try: From a3d21f06adaab181d566edbe6dbe250e5a0bff29 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 8 Feb 2012 10:46:33 -0800 Subject: [PATCH 304/334] Add auth checks to ec2 credential crud operations * Re-enables ec2 crud authorization tests * Fixes bug 928471 Change-Id: I22a97a8659ade5d146b52d112ff66ea58f847ef7 --- keystone/contrib/ec2/core.py | 58 ++++++++++++++++++++++++++++++------ tests/test_cli.py | 9 ++++++ tests/test_keystoneclient.py | 33 ++++++++++---------- 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index da493441af..90de2151a0 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -194,8 +194,8 @@ class Ec2Controller(wsgi.Application): :param tenant_id: id of tenant :returns: credential: dict of ec2 credential """ - # TODO(termie): validate that this request is valid for given user - # tenant + if not self._is_admin(context): + self._assert_identity(context, user_id) cred_ref = {'user_id': user_id, 'tenant_id': tenant_id, 'access': uuid.uuid4().hex, @@ -210,9 +210,8 @@ class Ec2Controller(wsgi.Application): :param user_id: id of user :returns: credentials: list of ec2 credential dicts """ - - # TODO(termie): validate that this request is valid for given user - # tenant + if not self._is_admin(context): + self._assert_identity(context, user_id) return {'credentials': self.ec2_api.list_credentials(context, user_id)} def get_credential(self, context, user_id, credential_id): @@ -225,8 +224,8 @@ class Ec2Controller(wsgi.Application): :param credential_id: access key for credentials :returns: credential: dict of ec2 credential """ - # TODO(termie): validate that this request is valid for given user - # tenant + if not self._is_admin(context): + self._assert_identity(context, user_id) return {'credential': self.ec2_api.get_credential(context, credential_id)} @@ -240,6 +239,47 @@ class Ec2Controller(wsgi.Application): :param credential_id: access key for credentials :returns: bool: success """ - # TODO(termie): validate that this request is valid for given user - # tenant + if not self._is_admin(context): + self._assert_identity(context, user_id) + self._assert_owner(context, user_id, credential_id) return self.ec2_api.delete_credential(context, credential_id) + + def _assert_identity(self, context, user_id): + """Check that the provided token belongs to the user. + + :param context: standard context + :param user_id: id of user + :raises webob.exc.HTTPForbidden: when token is invalid + + """ + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + token_user_id = token_ref['user'].get('id') + if not token_user_id == user_id: + raise webob.exc.HTTPForbidden() + + def _is_admin(self, context): + """Wrap admin assertion error return statement. + + :param context: standard context + :returns: bool: success + + """ + try: + self.assert_admin(context) + return True + except AssertionError: + return False + + def _assert_owner(self, context, user_id, credential_id): + """Ensure the provided user owns the credential. + + :param context: standard context + :param user_id: expected credential owner + :param credential_id: id of credential object + :raises webob.exc.HTTPForbidden: on failure + + """ + cred_ref = self.ec2_api.get_credential(context, credential_id) + if not user_id == cred_ref['user_id']: + raise webob.exc.HTTPForbidden() diff --git a/tests/test_cli.py b/tests/test_cli.py index bd4ead735b..2159e42167 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -76,3 +76,12 @@ class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): def test_service_create_and_delete(self): raise nose.exc.SkipTest('cli testing code does not handle 404 well') + + def test_ec2_credentials_list_user_forbidden(self): + raise nose.exc.SkipTest('cli testing code does not handle 403 well') + + def test_ec2_credentials_get_user_forbidden(self): + raise nose.exc.SkipTest('cli testing code does not handle 403 well') + + def test_ec2_credentials_delete_user_forbidden(self): + raise nose.exc.SkipTest('cli testing code does not handle 403 well') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 81d4175263..7f00b2a03e 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -285,45 +285,42 @@ class KeystoneClientTests(object): got = client.ec2.get(user_id=self.user_foo['id'], access=cred.access) self.assertEquals(cred, got) - # FIXME(ja): need to test ec2 validation here - client.ec2.delete(user_id=self.user_foo['id'], access=cred.access) creds = client.ec2.list(user_id=self.user_foo['id']) self.assertEquals(creds, []) - def test_ec2_credentials_list_unauthorized_user(self): - raise nose.exc.SkipTest('TODO') + def test_ec2_credentials_list_user_forbidden(self): from keystoneclient import exceptions as client_exceptions two = self.get_client(self.user_two) - self.assertRaises(client_exceptions.Unauthorized, two.ec2.list, - self.user_foo['id']) + self.assertRaises(client_exceptions.Forbidden, two.ec2.list, + user_id=self.user_foo['id']) - def test_ec2_credentials_get_unauthorized_user(self): - raise nose.exc.SkipTest('TODO') + def test_ec2_credentials_get_user_forbidden(self): from keystoneclient import exceptions as client_exceptions foo = self.get_client() - cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) + cred = foo.ec2.create(user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) two = self.get_client(self.user_two) - self.assertRaises(client_exceptions.Unauthorized, two.ec2.get, - self.user_foo['id'], cred.access) + self.assertRaises(client_exceptions.Forbidden, two.ec2.get, + user_id=self.user_foo['id'], access=cred.access) - foo.ec2.delete(self.user_foo['id'], cred.access) + foo.ec2.delete(user_id=self.user_foo['id'], access=cred.access) - def test_ec2_credentials_delete_unauthorized_user(self): - raise nose.exc.SkipTest('TODO') + def test_ec2_credentials_delete_user_forbidden(self): from keystoneclient import exceptions as client_exceptions foo = self.get_client() - cred = foo.ec2.create(self.user_foo['id'], self.tenant_bar['id']) + cred = foo.ec2.create(user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) two = self.get_client(self.user_two) - self.assertRaises(client_exceptions.Unauthorized, two.ec2.delete, - self.user_foo['id'], cred.access) + self.assertRaises(client_exceptions.Forbidden, two.ec2.delete, + user_id=self.user_foo['id'], access=cred.access) - foo.ec2.delete(self.user_foo['id'], cred.access) + foo.ec2.delete(user_id=self.user_foo['id'], access=cred.access) def test_service_create_and_delete(self): from keystoneclient import exceptions as client_exceptions From 95280608fc82f7ae51f8c42678d6a7f0ac2ecff0 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 8 Feb 2012 12:53:44 -0800 Subject: [PATCH 305/334] Cope with unicode passwords or None If the password has a unicode character in it, bcrypt breaks. So encode it using utf-8. utf-8 should mean that existing hashes still work. Change-Id: I4f9f3b636c8728234ada87de62d22bed2ff8eb60 --- keystone/common/utils.py | 10 +++++++--- tests/test_utils.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 tests/test_utils.py diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 16692bd635..6997eddd46 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -2,7 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara +# Copyright 2011 - 2012 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -144,7 +144,8 @@ class Ec2Signer(object): def hash_password(password): """Hash a password. Hard.""" salt = bcrypt.gensalt(CONF.bcrypt_strength) - return bcrypt.hashpw(password, salt) + password_utf8 = password.encode('utf-8') + return bcrypt.hashpw(password_utf8, salt) def check_password(password, hashed): @@ -155,7 +156,10 @@ def check_password(password, hashed): of that password (mostly). Neat! """ - check = bcrypt.hashpw(password, hashed[:29]) + if password is None: + return False + password_utf8 = password.encode('utf-8') + check = bcrypt.hashpw(password_utf8, hashed[:29]) return check == hashed diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000000..81cec7c918 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,40 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone import test +from keystone.common import utils + + +class UtilsTestCase(test.TestCase): + def test_hash(self): + password = 'right' + wrong = 'wrongwrong' # Two wrongs don't make a right + hashed = utils.hash_password(password) + self.assertTrue(utils.check_password(password, hashed)) + self.assertFalse(utils.check_password(wrong, hashed)) + + def test_hash_edge_cases(self): + hashed = utils.hash_password('secret') + self.assertFalse(utils.check_password('', hashed)) + self.assertFalse(utils.check_password(None, hashed)) + + def test_hash_unicode(self): + password = u'Comment \xe7a va' + wrong = 'Comment ?a va' + hashed = utils.hash_password(password) + self.assertTrue(utils.check_password(password, hashed)) + self.assertFalse(utils.check_password(wrong, hashed)) From 6013dd83bb96bc1f654fb8a0d7e7b29c25bfa36a Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 8 Feb 2012 13:22:51 -0800 Subject: [PATCH 306/334] Add content-type to responses * fixes bug 928055 Change-Id: Id86a9b3361d27493ed5ef175462aa1d4c1001bf4 --- keystone/common/wsgi.py | 9 ++++++--- tests/test_wsgi.py | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/test_wsgi.py diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index 34caec2c4d..b91052640f 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -177,10 +177,13 @@ class Application(BaseApplication): elif isinstance(result, webob.exc.WSGIHTTPException): return result - return self._serialize(result) + response = webob.Response() + self._serialize(response, result) + return response - def _serialize(self, result): - return json.dumps(result, cls=utils.SmarterEncoder) + def _serialize(self, response, result): + response.content_type = 'application/json' + response.body = json.dumps(result, cls=utils.SmarterEncoder) def _normalize_arg(self, arg): return str(arg).replace(':', '_').replace('-', '_') diff --git a/tests/test_wsgi.py b/tests/test_wsgi.py new file mode 100644 index 0000000000..cf329ba734 --- /dev/null +++ b/tests/test_wsgi.py @@ -0,0 +1,25 @@ +import webob + +from keystone import test +from keystone.common import wsgi + + +class FakeApp(wsgi.Application): + def index(self, context): + return {'a': 'b'} + + +class ApplicationTest(test.TestCase): + def setUp(self): + self.app = FakeApp() + + def _make_request(self): + req = webob.Request.blank('/') + args = {'action': 'index', 'controller': self.app} + req.environ['wsgiorg.routing_args'] = [None, args] + return req + + def test_response_content_type(self): + req = self._make_request() + resp = req.get_response(self.app) + self.assertEqual(resp.content_type, 'application/json') From c680d7ca5466e35b3b1c5c09dd949039c807ee8f Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 8 Feb 2012 12:31:25 -0800 Subject: [PATCH 307/334] Add SQL token backend * abstract out common token tests into test_backend * fixes bug 928052 Change-Id: I75da2f3c8d733b71025bcc1ef02f55e07645327f --- keystone/token/backends/sql.py | 52 ++++++++++++++++++++++++++++++++++ tests/test_backend.py | 19 +++++++++++++ tests/test_backend_kvs.py | 18 +----------- tests/test_backend_sql.py | 36 +++++++---------------- 4 files changed, 82 insertions(+), 43 deletions(-) create mode 100644 keystone/token/backends/sql.py diff --git a/keystone/token/backends/sql.py b/keystone/token/backends/sql.py new file mode 100644 index 0000000000..3bb5aea542 --- /dev/null +++ b/keystone/token/backends/sql.py @@ -0,0 +1,52 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone import token +from keystone.common import sql + + +class TokenModel(sql.ModelBase, sql.DictBase): + __tablename__ = 'token' + id = sql.Column(sql.String(64), primary_key=True) + extra = sql.Column(sql.JsonBlob()) + + @classmethod + def from_dict(cls, token_dict): + # shove any non-indexed properties into extra + data = {} + token_dict_copy = token_dict.copy() + data['id'] = token_dict_copy.pop('id') + data['extra'] = token_dict_copy + return cls(**data) + + def to_dict(self): + extra_copy = self.extra.copy() + extra_copy['id'] = self.id + return extra_copy + + +class Token(sql.Base, token.Driver): + # Public interface + def get_token(self, token_id): + session = self.get_session() + token_ref = session.query(TokenModel).filter_by(id=token_id).first() + if not token_ref: + return + return token_ref.to_dict() + + def create_token(self, token_id, data): + data['id'] = token_id + session = self.get_session() + with session.begin(): + token_ref = TokenModel.from_dict(data) + session.add(token_ref) + session.flush() + return token_ref.to_dict() + + def delete_token(self, token_id): + session = self.get_session() + token_ref = session.query(TokenModel)\ + .filter_by(id=token_id)\ + .first() + with session.begin(): + session.delete(token_ref) + session.flush() diff --git a/tests/test_backend.py b/tests/test_backend.py index b106f23f2c..59cebd6f31 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,3 +1,7 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import uuid + class IdentityTests(object): def test_authenticate_bad_user(self): self.assertRaises(AssertionError, @@ -197,3 +201,18 @@ class IdentityTests(object): self.assertEqual(tenant_ref['id'], 'fake1') tenant_ref = self.identity_api.get_tenant('fake2') self.assert_(tenant_ref is None) + + +class TokenTests(object): + def test_token_crud(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, 'a': 'b'} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + + new_data_ref = self.token_api.get_token(token_id) + self.assertEquals(new_data_ref, data) + + self.token_api.delete_token(token_id) + deleted_data_ref = self.token_api.get_token(token_id) + self.assertTrue(deleted_data_ref is None) diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index fe68dd78e4..4175360998 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -1,5 +1,3 @@ -import uuid - from keystone import test from keystone.identity.backends import kvs as identity_kvs from keystone.token.backends import kvs as token_kvs @@ -16,25 +14,11 @@ class KvsIdentity(test.TestCase, test_backend.IdentityTests): self.load_fixtures(default_fixtures) -class KvsToken(test.TestCase): +class KvsToken(test.TestCase, test_backend.TokenTests): def setUp(self): super(KvsToken, self).setUp() self.token_api = token_kvs.Token(db={}) - def test_token_crud(self): - token_id = uuid.uuid4().hex - data = {'id': token_id, - 'a': 'b'} - data_ref = self.token_api.create_token(token_id, data) - self.assertDictEquals(data_ref, data) - - new_data_ref = self.token_api.get_token(token_id) - self.assertEquals(new_data_ref, data) - - self.token_api.delete_token(token_id) - deleted_data_ref = self.token_api.get_token(token_id) - self.assert_(deleted_data_ref is None) - class KvsCatalog(test.TestCase): def setUp(self): diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py index fe137f525c..89068d7178 100644 --- a/tests/test_backend_sql.py +++ b/tests/test_backend_sql.py @@ -1,10 +1,10 @@ -import os -import uuid +# vim: tabstop=4 shiftwidth=4 softtabstop=4 from keystone import config from keystone import test from keystone.common.sql import util as sql_util from keystone.identity.backends import sql as identity_sql +from keystone.token.backends import sql as token_sql import test_backend import default_fixtures @@ -13,14 +13,9 @@ import default_fixtures CONF = config.CONF - class SqlIdentity(test.TestCase, test_backend.IdentityTests): def setUp(self): super(SqlIdentity, self).setUp() - try: - os.unlink('bla.db') - except Exception: - pass CONF(config_files=[test.etcdir('keystone.conf'), test.testsdir('test_overrides.conf'), test.testsdir('backend_sql.conf')]) @@ -29,25 +24,14 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests): self.load_fixtures(default_fixtures) -#class SqlToken(test_backend_kvs.KvsToken): -# def setUp(self): -# super(SqlToken, self).setUp() -# self.token_api = sql.SqlToken() -# self.load_fixtures(default_fixtures) - -# def test_token_crud(self): -# token_id = uuid.uuid4().hex -# data = {'id': token_id, -# 'a': 'b'} -# data_ref = self.token_api.create_token(token_id, data) -# self.assertDictEquals(data_ref, data) - -# new_data_ref = self.token_api.get_token(token_id) -# self.assertEquals(new_data_ref, data) - -# self.token_api.delete_token(token_id) -# deleted_data_ref = self.token_api.get_token(token_id) -# self.assert_(deleted_data_ref is None) +class SqlToken(test.TestCase, test_backend.TokenTests): + def setUp(self): + super(SqlToken, self).setUp() + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf'), + test.testsdir('backend_sql.conf')]) + sql_util.setup_test_database() + self.token_api = token_sql.Token() #class SqlCatalog(test_backend_kvs.KvsCatalog): From 26655dc7b7a5cf8374e1ecf4a9852e38a47be3b8 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 8 Feb 2012 11:50:34 -0800 Subject: [PATCH 308/334] Fix comment on bcrypt and avoid hard-coding 29 as the salt length Change-Id: Ifc78535ea79e071b7953769ff26eed8ecf666dc2 --- keystone/common/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 6997eddd46..43a4d420e4 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -151,15 +151,14 @@ def hash_password(password): def check_password(password, hashed): """Check that a plaintext password matches hashed. - Due to the way bcrypt works, hashing a password with the hashed - version of that password as salt will return the hashed version - of that password (mostly). Neat! + hashpw returns the salt value concatenated with the actual hash value. + It extracts the actual salt if this value is then passed as the salt. """ if password is None: return False password_utf8 = password.encode('utf-8') - check = bcrypt.hashpw(password_utf8, hashed[:29]) + check = bcrypt.hashpw(password_utf8, hashed) return check == hashed From c64a12ffc7072a5822d014da7111ed12adcf4046 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Tue, 7 Feb 2012 20:46:31 -0600 Subject: [PATCH 309/334] Friendly JSON exceptions (bug 928061, bug 928062) Example http://pastie.org/3338663 Change-Id: I26f53488c062ebfb6e49cfcf82e0b8179a683ea8 --- keystone/common/wsgi.py | 36 +++++++++++++++++++----- keystone/exception.py | 54 ++++++++++++++++++++++++++++++++++++ keystone/identity/core.py | 5 +++- keystone/service.py | 9 ++++++ tests/test_exception.py | 53 +++++++++++++++++++++++++++++++++++ tests/test_keystoneclient.py | 26 ++++++++++++++--- 6 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 keystone/exception.py create mode 100644 tests/test_exception.py diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index 34caec2c4d..c52df414ed 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -32,6 +32,7 @@ import webob import webob.dec import webob.exc +from keystone import exception from keystone.common import utils @@ -153,16 +154,13 @@ class Application(BaseApplication): @webob.dec.wsgify def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] - action = arg_dict['action'] - del arg_dict['action'] + action = arg_dict.pop('action') del arg_dict['controller'] logging.debug('arg_dict: %s', arg_dict) + # allow middleware up the stack to provide context & params context = req.environ.get('openstack.context', {}) - # allow middleware up the stack to override the params - params = {} - if 'openstack.params' in req.environ: - params = req.environ['openstack.params'] + params = req.environ.get('openstack.params', {}) params.update(arg_dict) # TODO(termie): do some basic normalization on methods @@ -170,7 +168,12 @@ class Application(BaseApplication): # NOTE(vish): make sure we have no unicode keys for py2.6. params = self._normalize_dict(params) - result = method(context, **params) + + try: + result = method(context, **params) + except exception.Error as e: + logging.warning(e) + return render_exception(e) if result is None or type(result) is str or type(result) is unicode: return result @@ -435,3 +438,22 @@ class ExtensionRouter(Router): conf.update(local_config) return cls(app) return _factory + + +def render_exception(error): + """Forms a WSGI response based on the current error.""" + resp = webob.Response() + resp.status = '%s %s' % (error.code, error.title) + resp.headerlist = [('Content-Type', 'application/json')] + + body = { + 'error': { + 'code': error.code, + 'title': error.title, + 'message': str(error), + } + } + + resp.body = json.dumps(body) + + return resp diff --git a/keystone/exception.py b/keystone/exception.py new file mode 100644 index 0000000000..cefc260431 --- /dev/null +++ b/keystone/exception.py @@ -0,0 +1,54 @@ +import re + + +class Error(StandardError): + """Base error class. + + Child classes should define an HTTP status code, title, and a doc string. + + """ + code = None + title = None + + def __init__(self, message=None, **kwargs): + """Use the doc string as the error message by default.""" + message = message or self.__doc__ % kwargs + super(Error, self).__init__(message) + + def __str__(self): + """Cleans up line breaks and indentation from doc strings.""" + string = super(Error, self).__str__() + string = re.sub('[ \n]+', ' ', string) + string = string.strip() + return string + + +class ValidationError(Error): + """Expecting to find %(attribute)s in %(target)s. + + The server could not comply with the request since it is either malformed + or otherwise incorrect. + + The client is assumed to be in error. + + """ + code = 400 + title = 'Bad Request' + + +class Unauthorized(Error): + """The request you have made requires authentication.""" + code = 401 + title = 'Not Authorized' + + +class Forbidden(Error): + """You are not authorized to perform the requested action: %(action)s""" + code = 403 + title = 'Not Authorized' + + +class NotFound(Error): + """Could not find: %(target)s""" + code = 404 + title = 'Not Found' diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 80ca31d121..5982a7ab00 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -10,6 +10,7 @@ import webob.exc from keystone import catalog from keystone import config +from keystone import exception from keystone import policy from keystone import token from keystone.common import manager @@ -233,7 +234,9 @@ class TenantController(wsgi.Application): """ token_ref = self.token_api.get_token(context=context, token_id=context['token_id']) - assert token_ref is not None + + if token_ref is None: + raise exception.Unauthorized() user_ref = token_ref['user'] tenant_ids = self.identity_api.get_tenants_for_user( diff --git a/keystone/service.py b/keystone/service.py index eae882d017..e2698cf7c1 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -10,6 +10,7 @@ import webob.dec import webob.exc from keystone import catalog +from keystone import exception from keystone import identity from keystone import policy from keystone import token @@ -196,6 +197,10 @@ class TokenController(wsgi.Application): old_token_ref = self.token_api.get_token(context=context, token_id=token) + + if old_token_ref is None: + raise exception.Unauthorized() + user_ref = old_token_ref['user'] tenants = self.identity_api.get_tenants_for_user(context, @@ -247,6 +252,10 @@ class TokenController(wsgi.Application): token_ref = self.token_api.get_token(context=context, token_id=token_id) + + if token_ref is None: + raise exception.NotFound(target='token') + if belongs_to: assert token_ref['tenant']['id'] == belongs_to diff --git a/tests/test_exception.py b/tests/test_exception.py new file mode 100644 index 0000000000..2c4b948406 --- /dev/null +++ b/tests/test_exception.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +import uuid +import json + +from keystone.common import wsgi +from keystone import exception +from keystone import test + + +class ExceptionTestCase(test.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def assertValidJsonRendering(self, e): + resp = wsgi.render_exception(e) + self.assertEqual(resp.status_int, e.code) + self.assertEqual(resp.status, '%s %s' % (e.code, e.title)) + + j = json.loads(resp.body) + self.assertIsNotNone(j.get('error')) + self.assertIsNotNone(j['error'].get('code')) + self.assertIsNotNone(j['error'].get('title')) + self.assertIsNotNone(j['error'].get('message')) + self.assertNotIn('\n', j['error']['message']) + self.assertNotIn(' ', j['error']['message']) + self.assertTrue(type(j['error']['code']) is int) + + def test_validation_error(self): + target = uuid.uuid4().hex + attribute = uuid.uuid4().hex + e = exception.ValidationError(target=target, attribute=attribute) + self.assertValidJsonRendering(e) + self.assertIn(target, str(e)) + self.assertIn(attribute, str(e)) + + def test_unauthorized(self): + e = exception.Unauthorized() + self.assertValidJsonRendering(e) + + def test_forbidden(self): + action = uuid.uuid4().hex + e = exception.Forbidden(action=action) + self.assertValidJsonRendering(e) + self.assertIn(action, str(e)) + + def test_not_found(self): + target = uuid.uuid4().hex + e = exception.NotFound(target=target) + self.assertValidJsonRendering(e) + self.assertIn(target, str(e)) diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 7f00b2a03e..036e62d8c5 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -125,6 +125,8 @@ class KeystoneClientTests(object): self.assertEquals(tenants[0].id, self.tenant_bar['id']) def test_authenticate_and_delete_token(self): + from keystoneclient import exceptions as client_exceptions + client = self.get_client() token = client.auth_token token_client = self._client(token=token) @@ -133,10 +135,26 @@ class KeystoneClientTests(object): client.tokens.delete(token_client.auth_token) - # FIXME(dolph): this should raise unauthorized - # from keystoneclient import exceptions as client_exceptions - # with self.assertRaises(client_exceptions.Unauthorized): - self.assertRaises(Exception, token_client.tenants.list) + self.assertRaises(client_exceptions.Unauthorized, + token_client.tenants.list) + + def test_authenticate_no_password(self): + from keystoneclient import exceptions as client_exceptions + + user_ref = self.user_foo.copy() + user_ref['password'] = None + self.assertRaises(client_exceptions.AuthorizationFailure, + self.get_client, + user_ref) + + def test_authenticate_no_username(self): + from keystoneclient import exceptions as client_exceptions + + user_ref = self.user_foo.copy() + user_ref['name'] = None + self.assertRaises(client_exceptions.AuthorizationFailure, + self.get_client, + user_ref) # TODO(termie): I'm not really sure that this is testing much def test_endpoints(self): From 05b2583dfa6d2f932f29e053c557b19601fae06b Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 6 Feb 2012 18:49:03 -0800 Subject: [PATCH 310/334] Add memcache token backend * tests use a fake memcache client * fixes bug 928040 Change-Id: I7c24e7829ee91fcf719eb4f338cf0ce2a0fa6bbd --- keystone/token/backends/memcache.py | 42 +++++++++++++++++++++++++++++ tests/test_backend_memcache.py | 33 +++++++++++++++++++++++ tools/pip-requires-test | 1 + 3 files changed, 76 insertions(+) create mode 100644 keystone/token/backends/memcache.py create mode 100644 tests/test_backend_memcache.py diff --git a/keystone/token/backends/memcache.py b/keystone/token/backends/memcache.py new file mode 100644 index 0000000000..686da5a71b --- /dev/null +++ b/keystone/token/backends/memcache.py @@ -0,0 +1,42 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from __future__ import absolute_import + +import memcache + +from keystone import config +from keystone import token + + +CONF = config.CONF +config.register_str('servers', group='memcache', default='localhost:11211') + + +class Token(token.Driver): + def __init__(self, client=None): + self._memcache_client = client + + @property + def client(self): + return self._memcache_client or self._get_memcache_client() + + def _get_memcache_client(self): + memcache_servers = CONF.memcache.servers.split(',') + self._memcache_client = memcache.Client(memcache_servers, debug=0) + return self._memcache_client + + def _prefix_token_id(self, token_id): + return 'token-%s' % token_id + + def get_token(self, token_id): + ptk = self._prefix_token_id(token_id) + return self.client.get(ptk) + + def create_token(self, token_id, data): + ptk = self._prefix_token_id(token_id) + self.client.set(ptk, data) + return data + + def delete_token(self, token_id): + ptk = self._prefix_token_id(token_id) + return self.client.delete(ptk) diff --git a/tests/test_backend_memcache.py b/tests/test_backend_memcache.py new file mode 100644 index 0000000000..61a967cc2c --- /dev/null +++ b/tests/test_backend_memcache.py @@ -0,0 +1,33 @@ +from keystone import test +from keystone.token.backends import memcache as token_memcache + + +class MemcacheClient(object): + """Replicates a tiny subset of memcached client interface.""" + + def __init__(self, *args, **kwargs): + """Ignores the passed in args.""" + self.cache = {} + + def get(self, key): + """Retrieves the value for a key or None.""" + return self.cache.get(key) + + def set(self, key, value): + """Sets the value for a key.""" + self.cache[key] = value + return True + + def delete(self, key): + try: + del self.cache[key] + except KeyError: + #NOTE(bcwaldon): python-memcached always returns the same value + pass + + +class MemcacheToken(test.TestCase): + def setUp(self): + super(MemcacheToken, self).setUp() + fake_client = MemcacheClient() + self.token_api = token_memcache.Token(client=fake_client) diff --git a/tools/pip-requires-test b/tools/pip-requires-test index 46beec26ff..f16a3b927d 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -9,6 +9,7 @@ pyCLI sqlalchemy sqlalchemy-migrate py-bcrypt +python-memcached # keystonelight testing dependencies nose From e5ffa7473359eee40c4644481067ef5212039150 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 9 Feb 2012 19:17:29 +0000 Subject: [PATCH 311/334] Fix largest memory leak in ksl tests * Explicitly kill wsgi servers that are launched * Fixes bug 929653 * Fix spaceypoo Change-Id: Id4b2f06749cb57c2680d37c1e4014c020d95ad5e --- keystone/common/wsgi.py | 7 ++++++- tests/test_keystoneclient.py | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index 59b49de242..ee5aac6bd6 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -55,6 +55,7 @@ class Server(object): self.port = port self.pool = eventlet.GreenPool(threads) self.socket_info = {} + self.greenthread = None def start(self, host='0.0.0.0', key=None, backlog=128): """Run a WSGI server with the given application.""" @@ -63,10 +64,14 @@ class Server(object): 'host': host, 'port': self.port}) socket = eventlet.listen((host, self.port), backlog=backlog) - self.pool.spawn_n(self._run, self.application, socket) + self.greenthread = self.pool.spawn(self._run, self.application, socket) if key: self.socket_info[key] = socket.getsockname() + def kill(self): + if self.greenthread: + self.greenthread.kill() + def wait(self): """Wait until all servers have completed running.""" try: diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 036e62d8c5..3f17cd9242 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -1,12 +1,10 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 import nose.exc -from keystone import config from keystone import test import default_fixtures -CONF = config.CONF OPENSTACK_REPO = 'https://review.openstack.org/p/openstack' KEYSTONECLIENT_REPO = '%s/python-keystoneclient.git' % OPENSTACK_REPO @@ -19,9 +17,6 @@ class CompatTestCase(test.TestCase): self.add_path(revdir) self.clear_module('keystoneclient') - self.public_app = self.loadapp('keystone', name='main') - self.admin_app = self.loadapp('keystone', name='admin') - self.load_backends() self.load_fixtures(default_fixtures) @@ -36,14 +31,19 @@ class CompatTestCase(test.TestCase): self.user_foo['id'], self.tenant_bar['id'], dict(roles=['keystone_admin'], is_admin='1')) + def tearDown(self): + self.public_server.kill() + self.admin_server.kill() + self.public_server = None + self.admin_server = None + super(CompatTestCase, self).tearDown() + def _public_url(self): public_port = self.public_server.socket_info['socket'][1] - CONF.public_port = public_port return "http://localhost:%s/v2.0" % public_port def _admin_url(self): admin_port = self.admin_server.socket_info['socket'][1] - CONF.admin_port = admin_port return "http://localhost:%s/v2.0" % admin_port def _client(self, **kwargs): @@ -473,10 +473,10 @@ class KcEssex3TestCase(CompatTestCase, KeystoneClientTests): roleref_refs = client.roles.get_user_role_refs( user_id=self.user_foo['id']) for roleref_ref in roleref_refs: - if (roleref_ref.roleId == self.role_useless['id'] and - roleref_ref.tenantId == self.tenant_baz['id']): - # use python's scope fall through to leave roleref_ref set - break + if (roleref_ref.roleId == self.role_useless['id'] + and roleref_ref.tenantId == self.tenant_baz['id']): + # use python's scope fall through to leave roleref_ref set + break client.roles.remove_user_from_tenant(tenant_id=self.tenant_baz['id'], user_id=self.user_foo['id'], From ae55fdca23135f76da4f7926135d384e9b13d419 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 9 Feb 2012 14:05:25 -0800 Subject: [PATCH 312/334] remove diablo tests, they aren't doing much Change-Id: I80640e123a784cd2ca5e9dfcf826a73666cea43c --- tests/test_legacy_compat.py | 170 ------------------------------------ 1 file changed, 170 deletions(-) delete mode 100644 tests/test_legacy_compat.py diff --git a/tests/test_legacy_compat.py b/tests/test_legacy_compat.py deleted file mode 100644 index e9d9ebc0f3..0000000000 --- a/tests/test_legacy_compat.py +++ /dev/null @@ -1,170 +0,0 @@ -import copy -import json -import os -import sys - -from nose import exc - -from keystone import config -from keystone import test -from keystone.common import logging -from keystone.common import utils - - -CONF = config.CONF - -OPENSTACK_REPO = 'https://review.openstack.org/p/openstack/' - -IDENTITY_API_REPO = '%s/identity-api.git' % OPENSTACK_REPO -KEYSTONE_REPO = '%s/keystone.git' % OPENSTACK_REPO -NOVACLIENT_REPO = '%s/python-novaclient.git' % OPENSTACK_REPO - -IDENTITY_SAMPLE_DIR = 'openstack-identity-api/src/docbkx/samples' -KEYSTONE_SAMPLE_DIR = 'keystone/content/common/samples' - - -class CompatTestCase(test.TestCase): - """Test compatibility against various versions of keystone's docs. - - It should be noted that the docs for any given revision have rarely, if ever, - reflected the actual usage or reliable sample output of the system, so these - tests are largely a study of frustration and its effects on developer - productivity. - - """ - - def setUp(self): - super(CompatTestCase, self).setUp() - - self.tenants_for_token = json.load(open( - os.path.join(self.sampledir, 'tenants.json'))) - self.validate_token = json.load(open( - os.path.join(self.sampledir, 'validatetoken.json'))) - # NOTE(termie): stupid hack to deal with the keystone samples being - # completely inconsistent - self.validate_token['access']['user']['roles'][1]['id'] = u'235' - self.admin_token = 'ADMIN' - - self.auth_response = json.load(open( - os.path.join(self.sampledir, 'auth.json'))) - - # validate_token call - self.tenant_345 = self.identity_api.create_tenant( - '345', - dict(id='345', name='My Project')) - self.user_123 = self.identity_api.create_user( - '123', - dict(id='123', - name='jqsmith', - tenants=[self.tenant_345['id']], - password='password')) - self.metadata_123 = self.identity_api.create_metadata( - self.user_123['id'], self.tenant_345['id'], - dict(roles=[{'id': '234', - 'name': 'compute:admin'}, - {'id': '235', - 'name': 'object-store:admin', - 'tenantId': '1'}], - roles_links=[])) - self.token_123 = self.token_api.create_token( - 'ab48a9efdfedb23ty3494', - dict(id='ab48a9efdfedb23ty3494', - expires='2010-11-01T03:32:15-05:00', - user=self.user_123, - tenant=self.tenant_345, - metadata=self.metadata_123)) - - # auth call - # NOTE(termie): the service catalog in the sample doesn't really have - # anything to do with the auth being returned, so just load - # it fully from a fixture and add it to our db - # NOTE(termie): actually all the data is insane anyway, so don't bother - #catalog = json.load(open( - # os.path.join(os.path.dirname(__file__), - # 'keystone_compat_diablo_sample_catalog.json'))) - #self.catalog_api.create_catalog(self.user_123['id'], - # self.tenant_345['id'], - # catalog) - - # tenants_for_token call - self.user_foo = self.identity_api.create_user( - 'foo', - dict(id='foo', name='FOO', tenants=['1234', '3456'])) - self.tenant_1234 = self.identity_api.create_tenant( - '1234', - dict(id='1234', - name='ACME Corp', - description='A description ...', - enabled=True)) - self.tenant_3456 = self.identity_api.create_tenant( - '3456', - dict(id='3456', - name='Iron Works', - description='A description ...', - enabled=True)) - - self.token_foo_unscoped = self.token_api.create_token( - 'foo_unscoped', - dict(id='foo_unscoped', - user=self.user_foo)) - self.token_foo_scoped = self.token_api.create_token( - 'foo_scoped', - dict(id='foo_scoped', - user=self.user_foo, - tenant=self.tenant_1234)) - - -class DiabloCompatTestCase(CompatTestCase): - def setUp(self): - CONF(config_files=[test.etcdir('keystone.conf'), - test.testsdir('test_overrides.conf')]) - - revdir = test.checkout_vendor(KEYSTONE_REPO, 'stable/diablo') - self.sampledir = os.path.join(revdir, KEYSTONE_SAMPLE_DIR) - self.app = self.loadapp('keystone') - - self.load_backends() - super(DiabloCompatTestCase, self).setUp() - - def test_authenticate_scoped(self): - # NOTE(termie): the docs arbitrarily changed and inserted a 'u' in front - # of one of the user ids, but none of the others - raise exc.SkipTest('The docs have arbitrarily changed.') - client = self.client(self.app) - post_data = json.dumps( - {'auth': {'passwordCredentials': {'username': self.user_123['id'], - 'password': self.user_123['password'], - }, - 'tenantName': self.tenant_345['name']}}) - - resp = client.post('/v2.0/tokens', body=post_data) - data = json.loads(resp.body) - logging.debug('KEYS: %s', data['access'].keys()) - self.assert_('expires' in data['access']['token']) - self.assertDeepEquals(self.auth_response['access']['user'], - data['access']['user']) - # there is pretty much no way to generate sane data that corresponds to - # the sample data - #self.assertDeepEquals(self.auth_response['access']['serviceCatalog'], - # data['access']['serviceCatalog']) - - def test_validate_token_scoped(self): - raise exc.SkipTest('The docs conflict with regular usage.') - client = self.client(self.app, token=self.admin_token) - resp = client.get('/v2.0/tokens/%s' % self.token_123['id']) - data = json.loads(resp.body) - self.assertDeepEquals(self.validate_token, data) - - def test_tenants_for_token_unscoped(self): - # get_tenants_for_token - client = self.client(self.app, token=self.token_foo_unscoped['id']) - resp = client.get('/v2.0/tenants') - data = json.loads(resp.body) - self.assertDeepEquals(self.tenants_for_token, data) - - def test_tenants_for_token_scoped(self): - # get_tenants_for_token - client = self.client(self.app, token=self.token_foo_scoped['id']) - resp = client.get('/v2.0/tenants') - data = json.loads(resp.body) - self.assertDeepEquals(self.tenants_for_token, data) From 2c18314e7cb7e5b7e5b6237f0793ec82739468f1 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 9 Feb 2012 09:53:03 -0800 Subject: [PATCH 313/334] Add TokenNotFound exception * raise TokenNotFound from token backends on get/delete when token doesn't exist Change-Id: Ic9aba7911088c30c20fe62501a05d75232f2d8b9 --- keystone/common/wsgi.py | 7 +++++-- keystone/contrib/ec2/core.py | 8 ++++++-- keystone/exception.py | 4 ++++ keystone/identity/core.py | 20 +++++--------------- keystone/service.py | 23 ++++++++++------------- keystone/token/backends/kvs.py | 13 ++++++++++--- keystone/token/backends/memcache.py | 9 ++++++++- keystone/token/backends/sql.py | 8 ++++++-- keystone/token/core.py | 4 +++- tests/test_backend.py | 9 +++++++-- tests/test_backend_memcache.py | 12 ++++++++++-- 11 files changed, 74 insertions(+), 43 deletions(-) diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index ee5aac6bd6..0c78e1d87a 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -202,8 +202,11 @@ class Application(BaseApplication): def assert_admin(self, context): if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) + try: + user_token_ref = self.token_api.get_token( + context=context, token_id=context['token_id']) + except exception.TokenNotFound: + raise exception.Unauthorized() creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index 888b8ddd3f..c8ad4425b6 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -26,6 +26,7 @@ import webob.exc from keystone import catalog from keystone import config +from keystone import exception from keystone import identity from keystone import policy from keystone import service @@ -252,8 +253,11 @@ class Ec2Controller(wsgi.Application): :raises webob.exc.HTTPForbidden: when token is invalid """ - token_ref = self.token_api.get_token(context=context, - token_id=context['token_id']) + try: + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + except exception.TokenNotFound: + raise exception.Unauthorized() token_user_id = token_ref['user'].get('id') if not token_user_id == user_id: raise webob.exc.HTTPForbidden() diff --git a/keystone/exception.py b/keystone/exception.py index cefc260431..c2f32afa30 100644 --- a/keystone/exception.py +++ b/keystone/exception.py @@ -52,3 +52,7 @@ class NotFound(Error): """Could not find: %(target)s""" code = 404 title = 'Not Found' + + +class TokenNotFound(NotFound): + """Could not find token: %(token_id)s""" diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 5982a7ab00..1928c6954b 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -232,10 +232,10 @@ class TenantController(wsgi.Application): Doesn't care about token scopedness. """ - token_ref = self.token_api.get_token(context=context, - token_id=context['token_id']) - - if token_ref is None: + try: + token_ref = self.token_api.get_token(context=context, + token_id=context['token_id']) + except exception.NotFound: raise exception.Unauthorized() user_ref = token_ref['user'] @@ -250,17 +250,7 @@ class TenantController(wsgi.Application): def get_tenant(self, context, tenant_id): # TODO(termie): this stuff should probably be moved to middleware - if not context['is_admin']: - user_token_ref = self.token_api.get_token( - context=context, token_id=context['token_id']) - creds = user_token_ref['metadata'].copy() - creds['user_id'] = user_token_ref['user'].get('id') - creds['tenant_id'] = user_token_ref['tenant'].get('id') - # Accept either is_admin or the admin role - assert self.policy_api.can_haz(context, - ('is_admin:1', 'roles:admin'), - creds) - + self.assert_admin(context) tenant = self.identity_api.get_tenant(context, tenant_id) if not tenant: return webob.exc.HTTPNotFound() diff --git a/keystone/service.py b/keystone/service.py index e2698cf7c1..0d94a5b906 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -1,8 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -import json -import urllib -import urlparse import uuid import routes @@ -15,7 +12,6 @@ from keystone import identity from keystone import policy from keystone import token from keystone.common import logging -from keystone.common import utils from keystone.common import wsgi @@ -195,10 +191,10 @@ class TokenController(wsgi.Application): else: tenant_id = auth.get('tenantId', None) - old_token_ref = self.token_api.get_token(context=context, - token_id=token) - - if old_token_ref is None: + try: + old_token_ref = self.token_api.get_token(context=context, + token_id=token) + except exception.NotFound: raise exception.Unauthorized() user_ref = old_token_ref['user'] @@ -253,9 +249,6 @@ class TokenController(wsgi.Application): token_ref = self.token_api.get_token(context=context, token_id=token_id) - if token_ref is None: - raise exception.NotFound(target='token') - if belongs_to: assert token_ref['tenant']['id'] == belongs_to @@ -277,8 +270,12 @@ class TokenController(wsgi.Application): def endpoints(self, context, token_id): """Return service catalog endpoints.""" - token_ref = self.token_api.get_token(context=context, - token_id=token_id) + try: + token_ref = self.token_api.get_token(context=context, + token_id=token_id) + except exception.NotFound: + raise exception.Unauthorized() + catalog_ref = self.catalog_api.get_catalog(context, token_ref['user']['id'], token_ref['tenant']['id']) diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py index c19bcc796e..84604ac5e3 100644 --- a/keystone/token/backends/kvs.py +++ b/keystone/token/backends/kvs.py @@ -1,17 +1,24 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -from keystone import token from keystone.common import kvs +from keystone import exception +from keystone import token class Token(kvs.Base, token.Driver): # Public interface def get_token(self, token_id): - return self.db.get('token-%s' % token_id) + try: + return self.db['token-%s' % token_id] + except KeyError: + raise exception.TokenNotFound(token_id=token_id) def create_token(self, token_id, data): self.db.set('token-%s' % token_id, data) return data def delete_token(self, token_id): - return self.db.delete('token-%s' % token_id) + try: + return self.db.delete('token-%s' % token_id) + except KeyError: + raise exception.TokenNotFound(token_id=token_id) diff --git a/keystone/token/backends/memcache.py b/keystone/token/backends/memcache.py index 686da5a71b..d4674ce9e5 100644 --- a/keystone/token/backends/memcache.py +++ b/keystone/token/backends/memcache.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import memcache from keystone import config +from keystone import exception from keystone import token @@ -30,7 +31,11 @@ class Token(token.Driver): def get_token(self, token_id): ptk = self._prefix_token_id(token_id) - return self.client.get(ptk) + token = self.client.get(ptk) + if token is None: + raise exception.TokenNotFound(token_id=token_id) + + return token def create_token(self, token_id, data): ptk = self._prefix_token_id(token_id) @@ -38,5 +43,7 @@ class Token(token.Driver): return data def delete_token(self, token_id): + # Test for existence + self.get_token(token_id) ptk = self._prefix_token_id(token_id) return self.client.delete(ptk) diff --git a/keystone/token/backends/sql.py b/keystone/token/backends/sql.py index 3bb5aea542..090a3600dd 100644 --- a/keystone/token/backends/sql.py +++ b/keystone/token/backends/sql.py @@ -1,7 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -from keystone import token from keystone.common import sql +from keystone import exception +from keystone import token class TokenModel(sql.ModelBase, sql.DictBase): @@ -30,7 +31,7 @@ class Token(sql.Base, token.Driver): session = self.get_session() token_ref = session.query(TokenModel).filter_by(id=token_id).first() if not token_ref: - return + raise exception.TokenNotFound(token_id=token_id) return token_ref.to_dict() def create_token(self, token_id, data): @@ -47,6 +48,9 @@ class Token(sql.Base, token.Driver): token_ref = session.query(TokenModel)\ .filter_by(id=token_id)\ .first() + if not token_ref: + raise exception.TokenNotFound(token_id=token_id) + with session.begin(): session.delete(token_ref) session.flush() diff --git a/keystone/token/core.py b/keystone/token/core.py index 47dadcea48..7183c1793e 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -29,7 +29,8 @@ class Driver(object): :param token_id: identity of the token :type token_id: string - :returns: token_ref or None. + :returns: token_ref + :raises: keystone.exception.TokenNotFound """ raise NotImplementedError() @@ -63,6 +64,7 @@ class Driver(object): :param token_id: identity of the token :type token_id: string :returns: None. + :raises: keystone.exception.TokenNotFound """ raise NotImplementedError() diff --git a/tests/test_backend.py b/tests/test_backend.py index 59cebd6f31..9dc949da57 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -2,6 +2,9 @@ import uuid +from keystone import exception + + class IdentityTests(object): def test_authenticate_bad_user(self): self.assertRaises(AssertionError, @@ -214,5 +217,7 @@ class TokenTests(object): self.assertEquals(new_data_ref, data) self.token_api.delete_token(token_id) - deleted_data_ref = self.token_api.get_token(token_id) - self.assertTrue(deleted_data_ref is None) + self.assertRaises(exception.TokenNotFound, + self.token_api.delete_token, token_id) + self.assertRaises(exception.TokenNotFound, + self.token_api.get_token, token_id) diff --git a/tests/test_backend_memcache.py b/tests/test_backend_memcache.py index 61a967cc2c..f665fb4bb5 100644 --- a/tests/test_backend_memcache.py +++ b/tests/test_backend_memcache.py @@ -1,6 +1,11 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +from keystone import exception from keystone import test from keystone.token.backends import memcache as token_memcache +import test_backend + class MemcacheClient(object): """Replicates a tiny subset of memcached client interface.""" @@ -11,7 +16,10 @@ class MemcacheClient(object): def get(self, key): """Retrieves the value for a key or None.""" - return self.cache.get(key) + try: + return self.cache[key] + except KeyError: + raise exception.TokenNotFound(token_id=key) def set(self, key, value): """Sets the value for a key.""" @@ -26,7 +34,7 @@ class MemcacheClient(object): pass -class MemcacheToken(test.TestCase): +class MemcacheToken(test.TestCase, test_backend.TokenTests): def setUp(self): super(MemcacheToken, self).setUp() fake_client = MemcacheClient() From 9028f3228b785cfefbbe3cd6532485817262c51d Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 9 Feb 2012 17:08:47 +0000 Subject: [PATCH 314/334] Add version description to root path * Fixes 925548 * Fix test for correct assertion * / returns {} Change-Id: I1067b402ad1bab474781e29ab7761f644f076540 --- etc/keystone.conf | 14 ++++++ keystone/service.py | 112 ++++++++++++++++++++++++++++++++--------- tests/test_versions.py | 107 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 25 deletions(-) create mode 100644 tests/test_versions.py diff --git a/etc/keystone.conf b/etc/keystone.conf index bbc84b78a2..fde1300487 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -68,10 +68,24 @@ pipeline = token_auth admin_token_auth json_body debug ec2_extension public_serv [pipeline:admin_api] pipeline = token_auth admin_token_auth json_body debug ec2_extension crud_extension admin_service +[app:public_version_service] +paste.app_factory = keystone.service:public_version_app_factory + +[app:admin_version_service] +paste.app_factory = keystone.service:admin_version_app_factory + +[pipeline:public_version_api] +pipeline = public_version_service + +[pipeline:admin_version_api] +pipeline = admin_version_service + [composite:main] use = egg:Paste#urlmap /v2.0 = public_api +/ = public_version_api [composite:admin] use = egg:Paste#urlmap /v2.0 = admin_api +/ = admin_version_service diff --git a/keystone/service.py b/keystone/service.py index e2698cf7c1..96e1cffa85 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -1,8 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -import json -import urllib -import urlparse import uuid import routes @@ -15,7 +12,6 @@ from keystone import identity from keystone import policy from keystone import token from keystone.common import logging -from keystone.common import utils from keystone.common import wsgi @@ -43,12 +39,6 @@ class AdminRouter(wsgi.ComposingRouter): conditions=dict(method=['GET'])) # Miscellaneous Operations - version_controller = VersionController() - mapper.connect('/', - controller=version_controller, - action='get_version_info', module='admin/version', - conditions=dict(method=['GET'])) - extensions_controller = ExtensionsController() mapper.connect('/extensions', controller=extensions_controller, @@ -76,13 +66,6 @@ class PublicRouter(wsgi.ComposingRouter): conditions=dict(method=['POST'])) # Miscellaneous - version_controller = VersionController() - mapper.connect('/', - controller=version_controller, - action='get_version_info', - module='service/version', - conditions=dict(method=['GET'])) - extensions_controller = ExtensionsController() mapper.connect('/extensions', controller=extensions_controller, @@ -95,6 +78,81 @@ class PublicRouter(wsgi.ComposingRouter): super(PublicRouter, self).__init__(mapper, routers) +class PublicVersionRouter(wsgi.ComposingRouter): + def __init__(self): + mapper = routes.Mapper() + version_controller = VersionController('public') + mapper.connect('/', + controller=version_controller, + action='get_versions') + routers = [] + super(PublicVersionRouter, self).__init__(mapper, routers) + + +class AdminVersionRouter(wsgi.ComposingRouter): + def __init__(self): + mapper = routes.Mapper() + version_controller = VersionController('admin') + mapper.connect('/', + controller=version_controller, + action='get_versions') + routers = [] + super(AdminVersionRouter, self).__init__(mapper, routers) + + +class VersionController(wsgi.Application): + def __init__(self, version_type): + self.catalog_api = catalog.Manager() + self.url_key = "%sURL" % version_type + super(VersionController, self).__init__() + + def _get_identity_url(self, context): + catalog_ref = self.catalog_api.get_catalog( + context=context, + user_id=None, + tenant_id=None) + for region, region_ref in catalog_ref.iteritems(): + for service, service_ref in region_ref.iteritems(): + if service == 'identity': + return service_ref[self.url_key] + + raise NotImplemented() + + def get_versions(self, context): + identity_url = self._get_identity_url(context) + if not identity_url.endswith('/'): + identity_url = identity_url + '/' + return { + "versions": { + "values": [{ + "id": "v2.0", + "status": "beta", + "updated": "2011-11-19T00:00:00Z", + "links": [{ + "rel": "self", + "href": identity_url, + }, { + "rel": "describedby", + "type": "text/html", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/content/" + }, { + "rel": "describedby", + "type": "application/pdf", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/identity-dev-guide-" + "2.0.pdf" + }], + "media-types": [{ + "base": "application/json", + "type": "application/vnd.openstack.identity-v2.0" + "+json" + }] + }] + } + } + + class NoopController(wsgi.Application): def __init__(self): super(NoopController, self).__init__() @@ -355,14 +413,6 @@ class TokenController(wsgi.Application): return services.values() -class VersionController(wsgi.Application): - def __init__(self): - super(VersionController, self).__init__() - - def get_version_info(self, context, module='version'): - raise NotImplemented() - - class ExtensionsController(wsgi.Application): def __init__(self): super(ExtensionsController, self).__init__() @@ -381,3 +431,15 @@ def admin_app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) return AdminRouter() + + +def public_version_app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return PublicVersionRouter() + + +def admin_version_app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return AdminVersionRouter() diff --git a/tests/test_versions.py b/tests/test_versions.py new file mode 100644 index 0000000000..6a08865498 --- /dev/null +++ b/tests/test_versions.py @@ -0,0 +1,107 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from keystone import test +from keystone import config + + +CONF = config.CONF + + +class VersionTestCase(test.TestCase): + def setUp(self): + super(VersionTestCase, self).setUp() + self.load_backends() + self.public_app = self.loadapp('keystone', 'main') + self.admin_app = self.loadapp('keystone', 'admin') + + self.public_server = self.serveapp('keystone', name='main') + self.admin_server = self.serveapp('keystone', name='admin') + + def test_public_versions(self): + client = self.client(self.public_app) + resp = client.get('/') + data = json.loads(resp.body) + expected = { + "versions": { + "values": [{ + "id": "v2.0", + "status": "beta", + "updated": "2011-11-19T00:00:00Z", + "links": [{ + "rel": "self", + "href": ("http://localhost:%s/v2.0/" % + CONF.public_port), + }, { + "rel": "describedby", + "type": "text/html", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/content/" + }, { + "rel": "describedby", + "type": "application/pdf", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/identity-dev-guide-" + "2.0.pdf" + }], + "media-types": [{ + "base": "application/json", + "type": "application/vnd.openstack.identity-v2.0" + "+json" + }] + }] + } + } + self.assertEqual(data, expected) + + def test_admin_versions(self): + client = self.client(self.admin_app) + resp = client.get('/') + data = json.loads(resp.body) + expected = { + "versions": { + "values": [{ + "id": "v2.0", + "status": "beta", + "updated": "2011-11-19T00:00:00Z", + "links": [{ + "rel": "self", + "href": ("http://localhost:%s/v2.0/" % + CONF.admin_port), + }, { + "rel": "describedby", + "type": "text/html", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/content/" + }, { + "rel": "describedby", + "type": "application/pdf", + "href": "http://docs.openstack.org/api/openstack-" + "identity-service/2.0/identity-dev-guide-" + "2.0.pdf" + }], + "media-types": [{ + "base": "application/json", + "type": "application/vnd.openstack.identity-v2.0" + "+json" + }] + }] + } + } + self.assertEqual(data, expected) From 363a5d63c0ff0630864765a07752cd63e6890e41 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 8 Feb 2012 14:31:41 -0800 Subject: [PATCH 315/334] Add tests for core middleware * Partially fixes bug 928039 Change-Id: I3807bcc77ab424c73069889b65b1a5598c17011c --- keystone/middleware/core.py | 10 +++-- tests/test_middleware.py | 76 +++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 tests/test_middleware.py diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index 8e206ddef4..c6324f3b9c 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -80,10 +80,11 @@ class JsonBodyMiddleware(wsgi.Middleware): an underscore. """ - def process_request(self, request): - #if 'json' not in request.params: - # return + # Ignore unrecognized content types. Empty string indicates + # the client did not explicitly set the header + if not request.content_type in ('application/json', ''): + return params_json = request.body if not params_json: @@ -92,6 +93,9 @@ class JsonBodyMiddleware(wsgi.Middleware): params_parsed = {} try: params_parsed = json.loads(params_json) + except ValueError: + msg = "Malformed json in request body" + raise webob.exc.HTTPBadRequest(explanation=msg) finally: if not params_parsed: params_parsed = {} diff --git a/tests/test_middleware.py b/tests/test_middleware.py new file mode 100644 index 0000000000..685853ab24 --- /dev/null +++ b/tests/test_middleware.py @@ -0,0 +1,76 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import webob + +from keystone import config +from keystone import middleware +from keystone import test + + +CONF = config.CONF + + +def make_request(**kwargs): + return webob.Request.blank('/', **kwargs) + + +class TokenAuthMiddlewareTest(test.TestCase): + def test_request(self): + req = make_request() + req.headers[middleware.AUTH_TOKEN_HEADER] = 'MAGIC' + middleware.TokenAuthMiddleware(None).process_request(req) + context = req.environ[middleware.CONTEXT_ENV] + self.assertEqual(context['token_id'], 'MAGIC') + + +class AdminTokenAuthMiddlewareTest(test.TestCase): + def test_request_admin(self): + req = make_request() + req.headers[middleware.AUTH_TOKEN_HEADER] = CONF.admin_token + middleware.AdminTokenAuthMiddleware(None).process_request(req) + context = req.environ[middleware.CONTEXT_ENV] + self.assertTrue(context['is_admin']) + + def test_request_non_admin(self): + req = make_request() + req.headers[middleware.AUTH_TOKEN_HEADER] = 'NOT-ADMIN' + middleware.AdminTokenAuthMiddleware(None).process_request(req) + context = req.environ[middleware.CONTEXT_ENV] + self.assertFalse(context['is_admin']) + + +class PostParamsMiddlewareTest(test.TestCase): + def test_request_with_params(self): + req = make_request(POST={"arg1": "one"}) + middleware.PostParamsMiddleware(None).process_request(req) + params = req.environ[middleware.PARAMS_ENV] + self.assertEqual(params, {"arg1": "one"}) + + +class JsonBodyMiddlewareTest(test.TestCase): + def test_request_with_params(self): + req = make_request(body='{"arg1": "one", "arg2": ["a"]}', + content_type='application/json') + middleware.JsonBodyMiddleware(None).process_request(req) + params = req.environ[middleware.PARAMS_ENV] + self.assertEqual(params, {"arg1": "one", "arg2": ["a"]}) + + def test_malformed_json(self): + req = make_request(body='{"arg1": "on', + content_type='application/json') + _middleware = middleware.JsonBodyMiddleware(None) + self.assertRaises(webob.exc.HTTPBadRequest, + _middleware.process_request, req) + + def test_no_content_type(self): + req = make_request(body='{"arg1": "one", "arg2": ["a"]}') + middleware.JsonBodyMiddleware(None).process_request(req) + params = req.environ[middleware.PARAMS_ENV] + self.assertEqual(params, {"arg1": "one", "arg2": ["a"]}) + + def test_unrecognized_content_type(self): + req = make_request(body='{"arg1": "one", "arg2": ["a"]}', + content_type='text/plain') + middleware.JsonBodyMiddleware(None).process_request(req) + params = req.environ.get(middleware.PARAMS_ENV, {}) + self.assertEqual(params, {}) From d049c19227d2702c5c5ac545d195465eac55246d Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 10 Feb 2012 14:23:12 -0800 Subject: [PATCH 316/334] Fix webob exceptions in test_middlware Change-Id: I211180ee0dde45f2030fba7e83fb1f6a1d880068 --- tests/test_middleware.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 685853ab24..f985850561 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -11,7 +11,13 @@ CONF = config.CONF def make_request(**kwargs): - return webob.Request.blank('/', **kwargs) + method = kwargs.pop('method', 'GET') + body = kwargs.pop('body', None) + req = webob.Request.blank('/', **kwargs) + req.method = method + if body is not None: + req.body = body + return req class TokenAuthMiddlewareTest(test.TestCase): @@ -41,7 +47,7 @@ class AdminTokenAuthMiddlewareTest(test.TestCase): class PostParamsMiddlewareTest(test.TestCase): def test_request_with_params(self): - req = make_request(POST={"arg1": "one"}) + req = make_request(body="arg1=one", method='POST') middleware.PostParamsMiddleware(None).process_request(req) params = req.environ[middleware.PARAMS_ENV] self.assertEqual(params, {"arg1": "one"}) @@ -50,27 +56,30 @@ class PostParamsMiddlewareTest(test.TestCase): class JsonBodyMiddlewareTest(test.TestCase): def test_request_with_params(self): req = make_request(body='{"arg1": "one", "arg2": ["a"]}', - content_type='application/json') + content_type='application/json', + method='POST') middleware.JsonBodyMiddleware(None).process_request(req) params = req.environ[middleware.PARAMS_ENV] self.assertEqual(params, {"arg1": "one", "arg2": ["a"]}) def test_malformed_json(self): req = make_request(body='{"arg1": "on', - content_type='application/json') + content_type='application/json', + method='POST') _middleware = middleware.JsonBodyMiddleware(None) self.assertRaises(webob.exc.HTTPBadRequest, _middleware.process_request, req) def test_no_content_type(self): - req = make_request(body='{"arg1": "one", "arg2": ["a"]}') + req = make_request(body='{"arg1": "one", "arg2": ["a"]}', method='POST') middleware.JsonBodyMiddleware(None).process_request(req) params = req.environ[middleware.PARAMS_ENV] self.assertEqual(params, {"arg1": "one", "arg2": ["a"]}) def test_unrecognized_content_type(self): req = make_request(body='{"arg1": "one", "arg2": ["a"]}', - content_type='text/plain') + content_type='text/plain', + method='POST') middleware.JsonBodyMiddleware(None).process_request(req) params = req.environ.get(middleware.PARAMS_ENV, {}) self.assertEqual(params, {}) From 79faa28f0389f427956ca72b3a902864ae5856f5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 8 Feb 2012 23:37:31 +0000 Subject: [PATCH 317/334] Fixes role checking for admin check Change-Id: I6afe52033996b56aa38033017e0ce2f37c471592 --- keystone/common/wsgi.py | 3 +++ keystone/middleware/core.py | 40 ++---------------------------- keystone/policy/backends/simple.py | 4 ++- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index ee5aac6bd6..f675ff87ec 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -207,6 +207,9 @@ class Application(BaseApplication): creds = user_token_ref['metadata'].copy() creds['user_id'] = user_token_ref['user'].get('id') creds['tenant_id'] = user_token_ref['tenant'].get('id') + # NOTE(vish): this is pretty inefficient + creds['roles'] = [self.identity_api.get_role(context, role)['name'] + for role in creds.get('roles', [])] # Accept either is_admin or the admin role assert self.policy_api.can_haz(context, ('is_admin:1', 'roles:admin'), diff --git a/keystone/middleware/core.py b/keystone/middleware/core.py index c6324f3b9c..09b86bbf6e 100644 --- a/keystone/middleware/core.py +++ b/keystone/middleware/core.py @@ -1,7 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 import json -import webob + +import webob.exc from keystone import config from keystone.common import wsgi @@ -109,40 +110,3 @@ class JsonBodyMiddleware(wsgi.Middleware): params[k] = v request.environ[PARAMS_ENV] = params - - -class Debug(wsgi.Middleware): - """ - Middleware that produces stream debugging traces to the console (stdout) - for HTTP requests and responses flowing through it. - """ - - @webob.dec.wsgify - def __call__(self, req): - print ('*' * 40) + ' REQUEST ENVIRON' - for key, value in req.environ.items(): - print key, '=', value - print - resp = req.get_response(self.application) - - print ('*' * 40) + ' RESPONSE HEADERS' - for (key, value) in resp.headers.iteritems(): - print key, '=', value - print - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """ - Iterator that prints the contents of a wrapper string iterator - when iterated. - """ - print ('*' * 40) + ' BODY' - for part in app_iter: - sys.stdout.write(part) - sys.stdout.flush() - yield part - print diff --git a/keystone/policy/backends/simple.py b/keystone/policy/backends/simple.py index 206ab574b1..ec4840fe4d 100644 --- a/keystone/policy/backends/simple.py +++ b/keystone/policy/backends/simple.py @@ -17,5 +17,7 @@ class SimpleMatch(object): for requirement in target: key, match = requirement.split(':', 1) check = credentials.get(key) - if check == match: + if check is None or isinstance(check, basestring): + check = [check] + if match in check: return True From 0e775d628b30f16a200d2b1f540d76ba24fbb351 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 9 Feb 2012 16:25:45 -0800 Subject: [PATCH 318/334] Add pagination to GET /tokens * Partially fixes bug 928049 Change-Id: I21943dcc7cea4dabfab672e84fe507e78e430de4 --- keystone/common/wsgi.py | 1 + keystone/identity/core.py | 31 ++++++++++++++++++-- tests/test_cli.py | 6 ++++ tests/test_keystoneclient.py | 55 ++++++++++++++++++++++++++++++++++++ tests/test_wsgi.py | 35 +++++++++++++++-------- 5 files changed, 114 insertions(+), 14 deletions(-) diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py index ee5aac6bd6..a5ae47fcbd 100644 --- a/keystone/common/wsgi.py +++ b/keystone/common/wsgi.py @@ -165,6 +165,7 @@ class Application(BaseApplication): # allow middleware up the stack to provide context & params context = req.environ.get('openstack.context', {}) + context['query_string'] = dict(req.params.iteritems()) params = req.environ.get('openstack.params', {}) params.update(arg_dict) diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 5982a7ab00..baa75947fc 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -246,7 +246,11 @@ class TenantController(wsgi.Application): tenant_refs.append(self.identity_api.get_tenant( context=context, tenant_id=tenant_id)) - return self._format_tenants_for_token(tenant_refs) + params = { + 'limit': context['query_string'].get('limit'), + 'marker': context['query_string'].get('marker'), + } + return self._format_tenant_list(tenant_refs, **params) def get_tenant(self, context, tenant_id): # TODO(termie): this stuff should probably be moved to middleware @@ -293,7 +297,30 @@ class TenantController(wsgi.Application): self.assert_admin(context) raise NotImplementedError() - def _format_tenants_for_token(self, tenant_refs): + def _format_tenant_list(self, tenant_refs, **kwargs): + marker = kwargs.get('marker') + page_idx = 0 + if marker is not None: + for (marker_idx, tenant) in enumerate(tenant_refs): + if tenant['id'] == marker: + # we start pagination after the marker + page_idx = marker_idx + 1 + break + else: + msg = 'Marker could not be found' + raise webob.exc.HTTPBadRequest(explanation=msg) + + limit = kwargs.get('limit') + if limit is not None: + try: + limit = int(limit) + assert limit >= 0 + except (ValueError, AssertionError): + msg = 'Invalid limit value' + raise webob.exc.HTTPBadRequest(explanation=msg) + + tenant_refs = tenant_refs[page_idx:limit] + for x in tenant_refs: x['enabled'] = True o = {'tenants': tenant_refs, diff --git a/tests/test_cli.py b/tests/test_cli.py index 2159e42167..04410d6e59 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -85,3 +85,9 @@ class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): def test_ec2_credentials_delete_user_forbidden(self): raise nose.exc.SkipTest('cli testing code does not handle 403 well') + + def test_tenant_list_limit_bad_value(self): + raise nose.exc.SkipTest('cli testing code does not handle 400 well') + + def test_tenant_list_marker_not_found(self): + raise nose.exc.SkipTest('cli testing code does not handle 400 well') diff --git a/tests/test_keystoneclient.py b/tests/test_keystoneclient.py index 3f17cd9242..1fca22fc79 100644 --- a/tests/test_keystoneclient.py +++ b/tests/test_keystoneclient.py @@ -1,4 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import uuid + import nose.exc from keystone import test @@ -448,6 +451,58 @@ class KcMasterTestCase(CompatTestCase, KeystoneClientTests): self.assert_(self.tenant_baz['id'] not in [x.id for x in tenant_refs]) + def test_tenant_list_marker(self): + client = self.get_client() + + # Add two arbitrary tenants to user for testing purposes + for i in range(2): + tenant_id = uuid.uuid4().hex + tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id} + self.identity_api.create_tenant(tenant_id, tenant) + self.identity_api.add_user_to_tenant(tenant_id, self.user_foo['id']) + + tenants = client.tenants.list() + self.assertEqual(len(tenants), 3) + + tenants_marker = client.tenants.list(marker=tenants[0].id) + self.assertEqual(len(tenants_marker), 2) + self.assertEqual(tenants[1].name, tenants_marker[0].name) + self.assertEqual(tenants[2].name, tenants_marker[1].name) + + def test_tenant_list_marker_not_found(self): + from keystoneclient import exceptions as client_exceptions + + client = self.get_client() + self.assertRaises(client_exceptions.BadRequest, + client.tenants.list, marker=uuid.uuid4().hex) + + def test_tenant_list_limit(self): + client = self.get_client() + + # Add two arbitrary tenants to user for testing purposes + for i in range(2): + tenant_id = uuid.uuid4().hex + tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id} + self.identity_api.create_tenant(tenant_id, tenant) + self.identity_api.add_user_to_tenant(tenant_id, self.user_foo['id']) + + tenants = client.tenants.list() + self.assertEqual(len(tenants), 3) + + tenants_limited = client.tenants.list(limit=2) + self.assertEqual(len(tenants_limited), 2) + self.assertEqual(tenants[0].name, tenants_limited[0].name) + self.assertEqual(tenants[1].name, tenants_limited[1].name) + + def test_tenant_list_limit_bad_value(self): + from keystoneclient import exceptions as client_exceptions + + client = self.get_client() + self.assertRaises(client_exceptions.BadRequest, + client.tenants.list, limit='a') + self.assertRaises(client_exceptions.BadRequest, + client.tenants.list, limit=-1) + def test_roles_get_by_user(self): client = self.get_client() roles = client.roles.roles_for_user(user=self.user_foo['id'], diff --git a/tests/test_wsgi.py b/tests/test_wsgi.py index cf329ba734..aa2c01223e 100644 --- a/tests/test_wsgi.py +++ b/tests/test_wsgi.py @@ -1,25 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import json + import webob from keystone import test from keystone.common import wsgi -class FakeApp(wsgi.Application): - def index(self, context): - return {'a': 'b'} - - class ApplicationTest(test.TestCase): - def setUp(self): - self.app = FakeApp() - - def _make_request(self): - req = webob.Request.blank('/') - args = {'action': 'index', 'controller': self.app} + def _make_request(self, url='/'): + req = webob.Request.blank(url) + args = {'action': 'index', 'controller': None} req.environ['wsgiorg.routing_args'] = [None, args] return req def test_response_content_type(self): + class FakeApp(wsgi.Application): + def index(self, context): + return {'a': 'b'} + + app = FakeApp() req = self._make_request() - resp = req.get_response(self.app) + resp = req.get_response(app) self.assertEqual(resp.content_type, 'application/json') + + def test_query_string_available(self): + class FakeApp(wsgi.Application): + def index(self, context): + return context['query_string'] + + app = FakeApp() + req = self._make_request(url='/?1=2') + resp = req.get_response(app) + self.assertEqual(json.loads(resp.body), {'1': '2'}) From af2836083b53c27406670841a57382e146868d02 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 2 Feb 2012 14:23:09 +0100 Subject: [PATCH 319/334] Add s3_token. Add s3_token middleware originally written by Akira YOSHIYAMA. Make it works with new swift_auth. Make not necessary modification to swift3 middleware. Handle errors when connecting to keystone. Address termie's comment in reviews. Change-Id: Icb2ae3fe79296ee6415dd55d146339ab72dd1d46 --- keystone/middleware/s3_token.py | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 keystone/middleware/s3_token.py diff --git a/keystone/middleware/s3_token.py b/keystone/middleware/s3_token.py new file mode 100644 index 0000000000..202e2c275a --- /dev/null +++ b/keystone/middleware/s3_token.py @@ -0,0 +1,136 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011,2012 Akira YOSHIYAMA +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This source code is based ./auth_token.py and ./ec2_token.py. +# See them for their copyright. + +"""Starting point for routing S3 requests.""" + +import httplib +import json + +import webob + +from swift.common import utils as swift_utils + + +PROTOCOL_NAME = "S3 Token Authentication" + + +class S3Token(object): + """Auth Middleware that handles S3 authenticating client calls.""" + + def __init__(self, app, conf): + """Common initialization code.""" + self.app = app + self.logger = swift_utils.get_logger(conf, log_route='s3_token') + self.logger.debug('Starting the %s component' % PROTOCOL_NAME) + + # where to find the auth service (we use this to validate tokens) + self.auth_host = conf.get('auth_host') + self.auth_port = int(conf.get('auth_port')) + self.auth_protocol = conf.get('auth_protocol', 'https') + + # where to tell clients to find the auth service (default to url + # constructed based on endpoint we have for the service to use) + self.auth_location = conf.get('auth_uri', + '%s://%s:%s' % (self.auth_protocol, + self.auth_host, + self.auth_port)) + + # Credentials used to verify this component with the Auth service since + # validating tokens is a privileged call + self.admin_token = conf.get('admin_token') + + def __call__(self, environ, start_response): + """Handle incoming request. authenticate and send downstream.""" + req = webob.Request(environ) + parts = swift_utils.split_path(req.path, 1, 4, True) + version, account, container, obj = parts + + # Read request signature and access id. + if not 'Authorization' in req.headers: + return self.app(environ, start_response) + token = req.headers.get('X-Auth-Token', + req.headers.get('X-Storage-Token')) + + auth_header = req.headers['Authorization'] + access, signature = auth_header.split(' ')[-1].rsplit(':', 1) + + # Authenticate the request. + creds = {'credentials': {'access': access, + 'token': token, + 'signature': signature, + 'host': req.host, + 'verb': req.method, + 'path': req.path, + 'expire': req.headers['Date'], + }} + + creds_json = json.dumps(creds) + headers = {'Content-Type': 'application/json'} + if self.auth_protocol == 'http': + conn = httplib.HTTPConnection(self.auth_host, self.auth_port) + else: + conn = httplib.HTTPSConnection(self.auth_host, self.auth_port) + + conn.request('POST', '/v2.0/s3tokens', + body=creds_json, + headers=headers) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise Exception('Keystone reply error: status=%s reason=%s' % ( + resp.status, + resp.reason)) + + # NOTE(vish): We could save a call to keystone by having + # keystone return token, tenant, user, and roles + # from this call. + # + # NOTE(chmou): We still have the same problem we would need to + # change token_auth to detect if we already + # identified and not doing a second query and just + # pass it through to swiftauth in this case. + # identity_info = json.loads(response) + output = resp.read() + conn.close() + identity_info = json.loads(output) + try: + token_id = str(identity_info['access']['token']['id']) + tenant = (identity_info['access']['token']['tenant']['id'], + identity_info['access']['token']['tenant']['name']) + except (KeyError, IndexError): + self.logger.debug('Error getting keystone reply: %s' % + (str(output))) + raise + + req.headers['X-Auth-Token'] = token_id + environ['PATH_INFO'] = environ['PATH_INFO'].replace( + account, 'AUTH_%s' % tenant[0]) + return self.app(environ, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return S3Token(app, conf) + return auth_filter From 6c5c964ff4b43457f5baafbbb889fffeccf92fc7 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 2 Feb 2012 13:15:26 +0000 Subject: [PATCH 320/334] Update swift token middleware. - Update swift middleware which add features: - Leaves authententication to token-auth. - ACL. - Container-Sync - Referer. - S3. - address termie comments on reviews. Change-Id: Iefaa228015d6286e4fb05be2cc24aec01589ef58 --- keystone/middleware/auth_token.py | 3 +- keystone/middleware/nova_auth_token.py | 3 +- keystone/middleware/swift_auth.py | 333 +++++++++++-------------- 3 files changed, 149 insertions(+), 190 deletions(-) mode change 100755 => 100644 keystone/middleware/swift_auth.py diff --git a/keystone/middleware/auth_token.py b/keystone/middleware/auth_token.py index 0dc723b9c6..4a0d501aa6 100755 --- a/keystone/middleware/auth_token.py +++ b/keystone/middleware/auth_token.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# + # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/keystone/middleware/nova_auth_token.py b/keystone/middleware/nova_auth_token.py index 68ad1d9db3..ad6bcbb810 100644 --- a/keystone/middleware/nova_auth_token.py +++ b/keystone/middleware/nova_auth_token.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# + # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/keystone/middleware/swift_auth.py b/keystone/middleware/swift_auth.py old mode 100755 new mode 100644 index e83c13667d..d12ac162fc --- a/keystone/middleware/swift_auth.py +++ b/keystone/middleware/swift_auth.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# -# Copyright (c) 2010-2011 OpenStack, LLC. + +# Copyright (c) 2012 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,220 +15,183 @@ # See the License for the specific language governing permissions and # limitations under the License. +import webob -""" -TOKEN-BASED AUTH MIDDLEWARE FOR SWIFT - -Authentication on incoming request - * grab token from X-Auth-Token header - * TODO: grab the memcache servers from the request env - * TODOcheck for auth information in memcache - * check for auth information from keystone - * return if unauthorized - * decorate the request for authorization in swift - * forward to the swift proxy app - -Authorization via callback - * check the path and extract the tenant - * get the auth information stored in keystone.identity during - authentication - * TODO: check if the user is an account admin or a reseller admin - * determine what object-type to authorize (account, container, object) - * use knowledge of tenant, admin status, and container acls to authorize - -""" - -import json -from urlparse import urlparse -from webob.exc import HTTPUnauthorized, HTTPNotFound, HTTPExpectationFailed - -from keystone.common.bufferedhttp import http_connect_raw as http_connect - -from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed -from swift.common.utils import get_logger, split_path +from swift.common import utils as swift_utils +from swift.common.middleware import acl as swift_acl -PROTOCOL_NAME = 'Swift Token Authentication' +class SwiftAuth(object): + """Swift middleware to Keystone authorization system. - -class AuthProtocol(object): - """Handles authenticating and aurothrizing client calls. - - Add to your pipeline in paste config like: + In Swift's proxy-server.conf add the middleware to your pipeline:: [pipeline:main] - pipeline = catch_errors healthcheck cache keystone proxy-server + pipeline = catch_errors cache tokenauth swiftauth proxy-server - [filter:keystone] + Set account auto creation to true:: + + [app:proxy-server] + account_autocreate = true + + And add a swift authorization filter section, such as:: + + [filter:swiftauth] use = egg:keystone#swiftauth - keystone_url = http://127.0.0.1:8080 - keystone_admin_token = 999888777666 + operator_roles = admin, SwiftOperator + is_admin = true + + If Swift memcache is to be used for caching tokens, add the additional + property in the tokenauth filter: + + [filter:tokenauth] + paste.filter_factory = keystone.middleware.auth_token:filter_factory + ... + cache = swift.cache + + This maps tenants to account in Swift. + + The user whose able to give ACL / create Containers permissions + will be the one that are inside the operator_roles + setting which by default includes the Admin and the SwiftOperator + roles. + + The option is_admin if set to true will allow the + username that has the same name as the account name to be the owner. + + Example: If we have the account called hellocorp with a user + hellocorp that user will be admin on that account and can give ACL + to all other users for hellocorp. + + :param app: The next WSGI app in the pipeline + :param conf: The dict of configuration values """ - def __init__(self, app, conf): - """Store valuable bits from the conf and set up logging.""" self.app = app - self.keystone_url = urlparse(conf.get('keystone_url')) - self.admin_token = conf.get('keystone_admin_token') - self.reseller_prefix = conf.get('reseller_prefix', 'AUTH') - self.log = get_logger(conf, log_route='keystone') - self.log.info('Keystone middleware started') + self.conf = conf + self.logger = swift_utils.get_logger(conf, log_route='keystoneauth') + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() + self.operator_roles = conf.get('operator_roles', + 'admin, SwiftOperator') + config_is_admin = conf.get('is_admin', "false").lower() + self.is_admin = config_is_admin in ('true', 't', '1', 'on', 'yes', 'y') + cfg_synchosts = conf.get('allowed_sync_hosts', '127.0.0.1') + self.allowed_sync_hosts = [h.strip() for h in cfg_synchosts.split(',') + if h.strip()] - def __call__(self, env, start_response): - """Authenticate the incoming request. + def __call__(self, environ, start_response): + identity = self._keystone_identity(environ) - If authentication fails return an appropriate http status here, - otherwise forward through the rest of the app. - """ + if not identity: + environ['swift.authorize'] = self.denied_response + return self.app(environ, start_response) - self.log.debug('Keystone middleware called') - token = self._get_claims(env) - self.log.debug('token: %s', token) - if token: - identity = self._validate_claims(token) - if identity: - self.log.debug('request authenticated: %r', identity) - return self.perform_authenticated_request(identity, env, - start_response) - else: - self.log.debug('anonymous request') - return self.unauthorized_request(env, start_response) - self.log.debug('no auth token in request headers') - return self.perform_unidentified_request(env, start_response) + self.logger.debug("Using identity: %r" % (identity)) + environ['keystone.identity'] = identity + environ['REMOTE_USER'] = identity.get('tenant') + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = swift_acl.clean_acl + return self.app(environ, start_response) - def unauthorized_request(self, env, start_response): - """Clinet provided a token that wasn't acceptable, error out.""" - return HTTPUnauthorized()(env, start_response) + def _keystone_identity(self, environ): + """Extract the identity from the Keystone auth component.""" + if environ.get('HTTP_X_IDENTITY_STATUS') != 'Confirmed': + return + roles = [] + if 'HTTP_X_ROLE' in environ: + roles = environ['HTTP_X_ROLE'].split(',') + identity = {'user': environ.get('HTTP_X_USER'), + 'tenant': (environ.get('HTTP_X_TENANT_ID'), + environ.get('HTTP_X_TENANT_NAME')), + 'roles': roles} + return identity - def unauthorized(self, req): - """Return unauthorized given a webob Request object. - - This can be stuffed into the evironment for swift.authorize or - called from the authoriztion callback when authorization fails. - """ - return HTTPUnauthorized(request=req) - - def perform_authenticated_request(self, identity, env, start_response): - """Client provieded a valid identity, so use it for authorization.""" - env['keystone.identity'] = identity - env['swift.authorize'] = self.authorize - env['swift.clean_acl'] = clean_acl - self.log.debug('calling app: %s // %r', start_response, env) - rv = self.app(env, start_response) - self.log.debug('return from app: %r', rv) - return rv - - def perform_unidentified_request(self, env, start_response): - """Withouth authentication data, use acls for access control.""" - env['swift.authorize'] = self.authorize_via_acl - env['swift.clean_acl'] = self.authorize_via_acl - return self.app(env, start_response) + def _reseller_check(self, account, tenant_id): + """Check reseller prefix.""" + return account == '%s_%s' % (self.reseller_prefix, tenant_id) def authorize(self, req): - """Used when we have a valid identity from keystone.""" - self.log.debug('keystone middleware authorization begin') env = req.environ - tenant = env.get('keystone.identity', {}).get('tenant') - if not tenant: - self.log.warn('identity info not present in authorize request') - return HTTPExpectationFailed('Unable to locate auth claim', - request=req) - # TODO(todd): everyone under a tenant can do anything to that tenant. - # more realistic would be role/group checking to do things - # like deleting the account or creating/deleting containers - # esp. when owned by other users in the same tenant. - if req.path.startswith('/v1/%s_%s' % (self.reseller_prefix, tenant)): - self.log.debug('AUTHORIZED OKAY') - return None + env_identity = env.get('keystone.identity', {}) + tenant = env_identity.get('tenant') - self.log.debug('tenant mismatch: %r', tenant) - return self.unauthorized(req) - - def authorize_via_acl(self, req): - """Anon request handling. - - For now this only allows anon read of objects. Container and account - actions are prohibited. - """ - - self.log.debug('authorizing anonymous request') try: - version, account, container, obj = split_path(req.path, 1, 4, True) + part = swift_utils.split_path(req.path, 1, 4, True) + version, account, container, obj = part except ValueError: - return HTTPNotFound(request=req) + return webob.exc.HTTPNotFound(request=req) - if obj: - return self._authorize_anon_object(req, account, container, obj) + if not self._reseller_check(account, tenant[0]): + log_msg = 'tenant mismatch: %s != %s' % (account, tenant[0]) + self.logger.debug(log_msg) + return self.denied_response(req) - if container: - return self._authorize_anon_container(req, account, container) + user_groups = env_identity.get('roles', []) - if account: - return self._authorize_anon_account(req, account) + # Check the groups the user is belonging to. If the user is + # part of the group defined in the config variable + # operator_roles (like Admin) then it will be + # promoted as an Admin of the account/tenant. + for group in self.operator_roles.split(','): + group = group.strip() + if group in user_groups: + log_msg = "allow user in group %s as account admin" % group + self.logger.debug(log_msg) + req.environ['swift_owner'] = True + return - return self._authorize_anon_toplevel(req) + # If user is of the same name of the tenant then make owner of it. + user = env_identity.get('user', '') + if self.is_admin and user == tenant[1]: + req.environ['swift_owner'] = True + return - def _authorize_anon_object(self, req, account, container, obj): - referrers, groups = parse_acl(getattr(req, 'acl', None)) - if referrer_allowed(req.referer, referrers): - self.log.debug('anonymous request AUTHORIZED OKAY') - return None - return self.unauthorized(req) + # Allow container sync. + if (req.environ.get('swift_sync_key') + and req.environ['swift_sync_key'] == + req.headers.get('x-container-sync-key', None) + and 'x-timestamp' in req.headers + and (req.remote_addr in self.allowed_sync_hosts + or swift_utils.get_remote_client(req) + in self.allowed_sync_hosts)): + log_msg = 'allowing proxy %s for container-sync' % req.remote_addr + self.logger.debug(log_msg) + return - def _authorize_anon_container(self, req, account, container): - return self.unauthorized(req) + # Check if referrer is allowed. + referrers, groups = swift_acl.parse_acl(getattr(req, 'acl', None)) + if swift_acl.referrer_allowed(req.referer, referrers): + if obj or '.rlistings' in groups: + log_msg = 'authorizing %s via referer ACL' % req.referrer + self.logger.debug(log_msg) + return + return self.denied_response(req) - def _authorize_anon_account(self, req, account): - return self.unauthorized(req) + # Allow ACL at individual user level (tenant:user format) + if '%s:%s' % (tenant[0], user) in groups: + log_msg = 'user %s:%s allowed in ACL authorizing' + self.logger.debug(log_msg % (tenant[0], user)) + return - def _authorize_anon_toplevel(self, req): - return self.unauthorized(req) + # Check if we have the group in the usergroups and allow it + for user_group in user_groups: + if user_group in groups: + log_msg = 'user %s:%s allowed in ACL: %s authorizing' + self.logger.debug(log_msg % (tenant[0], user, user_group)) + return - def _get_claims(self, env): - claims = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) - return claims + return self.denied_response(req) - def _validate_claims(self, claims): - """Ask keystone (as keystone admin) for information for this user.""" + def denied_response(self, req): + """Deny WSGI Response. - # TODO(todd): cache - - self.log.debug('Asking keystone to validate token') - headers = {'Content-type': 'application/json', - 'Accept': 'application/json', - 'X-Auth-Token': self.admin_token} - self.log.debug('headers: %r', headers) - self.log.debug('url: %s', self.keystone_url) - conn = http_connect(self.keystone_url.hostname, self.keystone_url.port, - 'GET', '/v2.0/tokens/%s' % claims, headers=headers) - resp = conn.getresponse() - data = resp.read() - conn.close() - - # Check http status code for the 'OK' family of responses - if not str(resp.status).startswith('20'): - return False - - identity_info = json.loads(data) - roles = [] - role_refs = identity_info['access']['user']['roles'] - - if role_refs is not None: - for role_ref in role_refs: - roles.append(role_ref['id']) - - try: - tenant = identity_info['access']['token']['tenantId'] - except: - tenant = None - if not tenant: - tenant = identity_info['access']['user']['tenantId'] - # TODO(Ziad): add groups back in - identity = {'user': identity_info['access']['user']['username'], - 'tenant': tenant, - 'roles': roles} - - return identity + Returns a standard WSGI response callable with the status of 403 or 401 + depending on whether the REMOTE_USER is set or not. + """ + if req.remote_user: + return webob.exc.HTTPForbidden(request=req) + else: + return webob.exc.HTTPUnauthorized(request=req) def filter_factory(global_conf, **local_conf): @@ -238,6 +200,5 @@ def filter_factory(global_conf, **local_conf): conf.update(local_conf) def auth_filter(app): - return AuthProtocol(app, conf) - + return SwiftAuth(app, conf) return auth_filter From 9f037224b0c1442f0d5a882d4f3ad400733aa59b Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Mon, 13 Feb 2012 22:39:23 +0000 Subject: [PATCH 321/334] Remove executable bit from auth_token.py Change-Id: I2422eb697ef41b74181bd6b4654d97ff6144bfd8 --- keystone/middleware/auth_token.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 keystone/middleware/auth_token.py diff --git a/keystone/middleware/auth_token.py b/keystone/middleware/auth_token.py old mode 100755 new mode 100644 From de8c9587277e2f12b4023ac7250f9c2e007db702 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 17:03:21 -0800 Subject: [PATCH 322/334] remove keystoneclient-based manage commands Change-Id: I2aaec673c2f2117026ae417b41b6996188f5bb84 --- keystone/cli.py | 257 +--------------------------------------------- tests/test_cli.py | 93 ----------------- 2 files changed, 1 insertion(+), 349 deletions(-) delete mode 100644 tests/test_cli.py diff --git a/keystone/cli.py b/keystone/cli.py index 9616f955db..1e47b5301a 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -8,7 +8,6 @@ import textwrap import cli.app import cli.log -from keystoneclient.v2_0 import client as kc from keystone import config from keystone.common import utils @@ -16,19 +15,6 @@ from keystone.common import utils CONF = config.CONF CONF.set_usage('%prog COMMAND [key1=value1 key2=value2 ...]') -config.register_cli_str('endpoint', - default='http://localhost:$admin_port/v2.0', - #group='ks', - conf=CONF) -config.register_cli_str('auth-token', - default='$admin_token', - #group='ks', - help='authorization token', - conf=CONF) -config.register_cli_bool('id-only', - default=False, - #group='ks', - conf=CONF) class BaseApp(cli.log.LoggingApp): @@ -70,249 +56,8 @@ class DbSync(BaseApp): driver.db_sync() -class ClientCommand(BaseApp): - ACTION_MAP = None - - def _attr_name(self): - return '%ss' % self.__class__.__name__.lower() - - def _cmd_name(self): - return self.__class__.__name__.lower() - - def __init__(self, *args, **kw): - super(ClientCommand, self).__init__(*args, **kw) - if not self.ACTION_MAP: - self.ACTION_MAP = {'help': 'help'} - self.add_param('action', nargs='?', default='help') - self.add_param('keyvalues', nargs='*') - self.client = kc.Client(CONF.endpoint, token=CONF.auth_token) - self.handle = getattr(self.client, self._attr_name()) - self._build_action_map() - - def _build_action_map(self): - actions = {} - for k in dir(self.handle): - if not k.startswith('_'): - actions[k] = k - self.ACTION_MAP.update(actions) - - def main(self): - """Given some keyvalues create the appropriate data in Keystone.""" - action_name = self.ACTION_MAP[self.params.action] - if action_name == 'help': - self.print_help() - sys.exit(1) - - kv = self._parse_keyvalues(self.params.keyvalues) - try: - f = getattr(self.handle, action_name) - resp = f(**kv) - except Exception: - logging.exception('') - raise - - if CONF.id_only and getattr(resp, 'id'): - print resp.id - return - - if resp is None: - return - - # NOTE(termie): this is ugly but it is mostly because the - # keystoneclient code doesn't give us very - # serializable instance objects - if type(resp) in [type(list()), type(tuple())]: - o = [] - for r in resp: - d = {} - for k, v in sorted(r.__dict__.iteritems()): - if k[0] == '_' or k == 'manager': - continue - d[k] = v - o.append(d) - else: - o = {} - for k, v in sorted(resp.__dict__.iteritems()): - if k[0] == '_' or k == 'manager': - continue - o[k] = v - - print json.dumps(o) - - def print_help(self): - CONF.set_usage(CONF.usage.replace( - 'COMMAND', '%s SUBCOMMAND' % self._cmd_name())) - CONF.print_help() - - methods = self._get_methods() - print_commands(methods) - - def _get_methods(self): - o = {} - for k in dir(self.handle): - if k.startswith('_'): - continue - if k in ('find', 'findall', 'api', 'resource_class'): - continue - o[k] = getattr(self.handle, k) - return o - - -class Role(ClientCommand): - """Role CRUD functions.""" - pass - - -class Service(ClientCommand): - """Service CRUD functions.""" - pass - - -class Token(ClientCommand): - """Token CRUD functions.""" - pass - - -class Tenant(ClientCommand): - """Tenant CRUD functions.""" - pass - - -class User(ClientCommand): - """User CRUD functions.""" - - pass - - -class Ec2(ClientCommand): - def _attr_name(self): - return self.__class__.__name__.lower() - - CMDS = {'db_sync': DbSync, - 'role': Role, - 'service': Service, - 'token': Token, - 'tenant': Tenant, - 'user': User, - 'ec2': Ec2, - } - - -class CommandLineGenerator(object): - """A keystoneclient lookalike to generate keystone-manage commands. - - One would use it like so: - - >>> gen = CommandLineGenerator(id_only=None) - >>> cl = gen.ec2.create(user_id='foo', tenant_id='foo') - >>> cl.to_argv() - ... ['keystone-manage', - '--id-only', - 'ec2', - 'create', - 'user_id=foo', - 'tenant_id=foo'] - - """ - - cmd = 'keystone-manage' - - def __init__(self, cmd=None, execute=False, **kw): - if cmd: - self.cmd = cmd - self.flags = kw - self.execute = execute - - def __getattr__(self, key): - return _Manager(self, key) - - -class _Manager(object): - def __init__(self, parent, name): - self.parent = parent - self.name = name - - def __getattr__(self, key): - return _CommandLine(cmd=self.parent.cmd, - flags=self.parent.flags, - manager=self.name, - method=key, - execute=self.parent.execute) - - -class _CommandLine(object): - def __init__(self, cmd, flags, manager, method, execute=False): - self.cmd = cmd - self.flags = flags - self.manager = manager - self.method = method - self.execute = execute - self.kw = {} - - def __call__(self, **kw): - self.kw = kw - if self.execute: - logging.debug('generated cli: %s', str(self)) - out = StringIO.StringIO() - old_out = sys.stdout - sys.stdout = out - try: - main(self.to_argv()) - except SystemExit as e: - pass - finally: - sys.stdout = old_out - rv = out.getvalue().strip().split('\n')[-1] - try: - loaded = json.loads(rv) - if type(loaded) in [type(list()), type(tuple())]: - return [DictWrapper(**x) for x in loaded] - elif type(loaded) is type(dict()): - return DictWrapper(**loaded) - except Exception: - logging.exception('Could not parse JSON: %s', rv) - return rv - return self - - def __flags(self): - o = [] - for k, v in self.flags.iteritems(): - k = k.replace('_', '-') - if v is None: - o.append('--%s' % k) - else: - o.append('--%s=%s' % (k, str(v))) - return o - - def __manager(self): - if self.manager.endswith('s'): - return self.manager[:-1] - return self.manager - - def __kw(self): - o = [] - for k, v in self.kw.iteritems(): - o.append('%s=%s' % (k, str(v))) - return o - - def to_argv(self): - return ([self.cmd] - + self.__flags() - + [self.__manager(), self.method] - + self.__kw()) - - def __str__(self): - args = self.to_argv() - return ' '.join(args[:1] + ['"%s"' % x for x in args[1:]]) - - -class DictWrapper(dict): - def __getattr__(self, key): - try: - return self[key] - except KeyError: - raise AttributeError(key) + } def print_commands(cmds): diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 04410d6e59..0000000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,93 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 -import nose.exc - -from keystone import config -from keystone import test -from keystone.common import utils - -import default_fixtures -import test_keystoneclient - -CONF = config.CONF -KEYSTONECLIENT_REPO = 'git://github.com/openstack/python-keystoneclient.git' - - -class CliMasterTestCase(test_keystoneclient.KcMasterTestCase): - def setUp(self): - super(CliMasterTestCase, self).setUp() - # NOTE(termie): we need to reset and reparse the config here because - # cli adds new command-line config options - # NOTE(termie): we are importing cli here because it imports - # keystoneclient, which we are loading from different - # sources between tests - CONF.reset() - from keystone import cli - self.cli = cli - self.config() - - def get_client(self, user_ref=None, tenant_ref=None): - if user_ref is None: - user_ref = self.user_foo - if tenant_ref is None: - for user in default_fixtures.USERS: - if user['id'] == user_ref['id']: - tenant_id = user['tenants'][0] - else: - tenant_id = tenant_ref['id'] - - cl = self._client(username=user_ref['name'], - password=user_ref['password'], - tenant_id=tenant_id) - gen = self.cli.CommandLineGenerator( - cmd=test.rootdir('bin', 'keystone-manage'), - execute=True, - auth_token=cl.auth_token, - endpoint=cl.management_url) - gen.auth_token = cl.auth_token - gen.management_url = cl.management_url - return gen - - def test_authenticate_tenant_id_and_tenants(self): - raise nose.exc.SkipTest('N/A') - - def test_authenticate_token_no_tenant(self): - raise nose.exc.SkipTest('N/A') - - def test_authenticate_token_tenant_id(self): - raise nose.exc.SkipTest('N/A') - - def test_authenticate_token_tenant_name(self): - raise nose.exc.SkipTest('N/A') - - def test_authenticate_and_delete_token(self): - raise nose.exc.SkipTest('N/A') - - def test_tenant_create_update_and_delete(self): - raise nose.exc.SkipTest('cli does not support booleans yet') - - def test_invalid_password(self): - raise nose.exc.SkipTest('N/A') - - def test_user_create_update_delete(self): - raise nose.exc.SkipTest('cli does not support booleans yet') - - def test_role_create_and_delete(self): - raise nose.exc.SkipTest('cli testing code does not handle 404 well') - - def test_service_create_and_delete(self): - raise nose.exc.SkipTest('cli testing code does not handle 404 well') - - def test_ec2_credentials_list_user_forbidden(self): - raise nose.exc.SkipTest('cli testing code does not handle 403 well') - - def test_ec2_credentials_get_user_forbidden(self): - raise nose.exc.SkipTest('cli testing code does not handle 403 well') - - def test_ec2_credentials_delete_user_forbidden(self): - raise nose.exc.SkipTest('cli testing code does not handle 403 well') - - def test_tenant_list_limit_bad_value(self): - raise nose.exc.SkipTest('cli testing code does not handle 400 well') - - def test_tenant_list_marker_not_found(self): - raise nose.exc.SkipTest('cli testing code does not handle 400 well') From eb5a939b4691da8b433043259b1541686285bb35 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 15:18:35 -0800 Subject: [PATCH 323/334] add migration from legacy db Change-Id: If49d829a36c61d6ee379f35c793196fde4ab4657 --- keystone/common/sql/legacy.py | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 keystone/common/sql/legacy.py diff --git a/keystone/common/sql/legacy.py b/keystone/common/sql/legacy.py new file mode 100644 index 0000000000..1d6c7d6151 --- /dev/null +++ b/keystone/common/sql/legacy.py @@ -0,0 +1,113 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import sqlalchemy + +from keystone.identity.backends import sql as identity_sql + + +def export_db(db): + table_names = db.table_names() + + migration_data = {} + for table_name in table_names: + query = db.execute("select * from %s" % table_name) + table_data = [] + for row in query.fetchall(): + entry = {} + for k in row.keys(): + entry[k] = row[k] + table_data.append(entry) + + migration_data[table_name] = table_data + + return migration_data + + +class LegacyMigration(object): + def __init__(self, db_string): + self.db = sqlalchemy.create_engine(db_string) + self.identity_driver = identity_sql.Identity() + self.identity_driver.db_sync() + self._data = {} + self._user_map = {} + self._tenant_map = {} + self._role_map = {} + + def migrate_all(self): + self._export_legacy_db() + self._migrate_tenants() + self._migrate_users() + self._migrate_roles() + self._migrate_user_roles() + self._migrate_tokens() + + def dump_catalog(self, path): + """Generate the contents of a catalog templates file.""" + pass + + def _export_legacy_db(self): + self._data = export_db(self.db) + + def _migrate_tenants(self): + for x in self._data['tenants']: + # map + new_dict = {'description': x.get('desc', ''), + 'id': x.get('uid', x.get('id')), + 'enabled': x.get('enabled', True)} + new_dict['name'] = x.get('name', new_dict.get('id')) + # track internal ids + self._tenant_map[x.get('id')] = new_dict['id'] + # create + #print 'create_tenant(%s, %s)' % (new_dict['id'], new_dict) + self.identity_driver.create_tenant(new_dict['id'], new_dict) + + def _migrate_users(self): + for x in self._data['users']: + # map + new_dict = {'email': x.get('email', ''), + 'password': x.get('password', None), + 'id': x.get('uid', x.get('id')), + 'enabled': x.get('enabled', True)} + if x.get('tenant_id'): + new_dict['tenant_id'] = self._tenant_map.get(x['tenant_id']) + new_dict['name'] = x.get('name', new_dict.get('id')) + # track internal ids + self._user_map[x.get('id')] = new_dict['id'] + # create + #print 'create_user(%s, %s)' % (new_dict['id'], new_dict) + self.identity_driver.create_user(new_dict['id'], new_dict) + if new_dict.get('tenant_id'): + self.identity_driver.add_user_to_tenant(new_dict['tenant_id'], + new_dict['id']) + + def _migrate_roles(self): + for x in self._data['roles']: + # map + new_dict = {'id': x['id'], + 'name': x.get('name', x['id'])} + # track internal ids + self._role_map[x.get('id')] = new_dict['id'] + # create + self.identity_driver.create_role(new_dict['id'], new_dict) + + def _migrate_user_roles(self): + for x in self._data['user_roles']: + # map + if (not x.get('user_id') + or not x.get('tenant_id') + or not x.get('role_id')): + continue + user_id = self._user_map[x['user_id']] + tenant_id = self._tenant_map[x['tenant_id']] + role_id = self._role_map[x['role_id']] + + try: + self.identity_driver.add_user_to_tenant(tenant_id, user_id) + except Exception: + pass + + self.identity_driver.add_role_to_user_and_tenant( + user_id, tenant_id, role_id) + + def _migrate_tokens(self): + pass From 63adca31f772ab50a2da1c81bda9a66e0e009e02 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 16:50:00 -0800 Subject: [PATCH 324/334] add import legacy cli command Change-Id: I41f0baaf3e7beb5c1ba44a7da420b411c1f60277 --- keystone/cli.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/keystone/cli.py b/keystone/cli.py index 1e47b5301a..b7e7dfbbff 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + from __future__ import absolute_import import json @@ -56,7 +58,24 @@ class DbSync(BaseApp): driver.db_sync() +class ImportLegacy(BaseApp): + """Import a legacy database.""" + + name = 'import_legacy' + + def __init__(self, *args, **kw): + super(ImportLegacy, self).__init__(*args, **kw) + self.add_param('old_db', nargs=1) + + def main(self): + from keystone.common.sql import legacy + old_db = self.params.old_db[0] + migration = legacy.LegacyMigration(old_db) + migration.migrate_all() + + CMDS = {'db_sync': DbSync, + 'import_legacy': ImportLegacy, } From 700a397a64bf984ef4c56aec8cc597f212e1f459 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 17:01:02 -0800 Subject: [PATCH 325/334] add sql for import legacy tests Change-Id: I778300970c12bff0c296e6551c1b4e9e079496dc --- tests/legacy_d5.mysql | 281 +++++++++++++++++++++++++++++++++++++ tests/legacy_d5.sqlite | 277 ++++++++++++++++++++++++++++++++++++ tools/convert_to_sqlite.sh | 71 ++++++++++ 3 files changed, 629 insertions(+) create mode 100644 tests/legacy_d5.mysql create mode 100644 tests/legacy_d5.sqlite create mode 100755 tools/convert_to_sqlite.sh diff --git a/tests/legacy_d5.mysql b/tests/legacy_d5.mysql new file mode 100644 index 0000000000..57b31febe5 --- /dev/null +++ b/tests/legacy_d5.mysql @@ -0,0 +1,281 @@ +-- MySQL dump 10.13 Distrib 5.1.54, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.54-1ubuntu4 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `user_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `type` varchar(20) DEFAULT NULL, + `key` varchar(255) DEFAULT NULL, + `secret` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `tenant_id` (`tenant_id`), + KEY `user_id` (`user_id`) +) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + +LOCK TABLES `credentials` WRITE; +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +INSERT INTO `credentials` VALUES (1,1,1,'EC2','admin','secrete'),(2,2,2,'EC2','demo','secrete'); +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `region` varchar(255) DEFAULT NULL, + `service_id` int(11) DEFAULT NULL, + `public_url` varchar(2000) DEFAULT NULL, + `admin_url` varchar(2000) DEFAULT NULL, + `internal_url` varchar(2000) DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `is_global` tinyint(1) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `service_id` (`service_id`) +) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + +LOCK TABLES `endpoint_templates` WRITE; +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne',1,'http://10.4.128.10:8774/v1.1/%tenant_id%','http://10.4.128.10:8774/v1.1/%tenant_id%','http://10.4.128.10:8774/v1.1/%tenant_id%',1,1),(2,'RegionOne',2,'http://10.4.128.10:9292/v1.1/%tenant_id%','http://10.4.128.10:9292/v1.1/%tenant_id%','http://10.4.128.10:9292/v1.1/%tenant_id%',1,1),(3,'RegionOne',3,'http://10.4.128.10:5000/v2.0','http://10.4.128.10:35357/v2.0','http://10.4.128.10:5000/v2.0',1,1); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tenant_id` int(11) DEFAULT NULL, + `endpoint_template_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `endpoint_template_id` (`endpoint_template_id`,`tenant_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + +LOCK TABLES `endpoints` WRITE; +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + `service_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`,`service_id`), + KEY `service_id` (`service_id`) +) ENGINE=MyISAM AUTO_INCREMENT=5 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + +LOCK TABLES `roles` WRITE; +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (1,'Admin',NULL,NULL),(2,'Member',NULL,NULL),(3,'KeystoneAdmin',NULL,NULL),(4,'KeystoneServiceAdmin',NULL,NULL); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `type` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + +LOCK TABLES `services` WRITE; +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES (1,'nova','compute','Nova Compute Service'),(2,'glance','image','Glance Image Service'),(3,'keystone','identity','Keystone Identity Service'); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + `enabled` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + +LOCK TABLES `tenants` WRITE; +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (1,'admin',NULL,1),(2,'demo',NULL,1),(3,'invisible_to_admin',NULL,1); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `token` +-- + +DROP TABLE IF EXISTS `token`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `token` ( + `id` varchar(255) NOT NULL, + `user_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `expires` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `token` +-- + +LOCK TABLES `token` WRITE; +/*!40000 ALTER TABLE `token` DISABLE KEYS */; +INSERT INTO `token` VALUES ('secrete',1,1,'2015-02-05 00:00:00'); +/*!40000 ALTER TABLE `token` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `user_id` int(11) DEFAULT NULL, + `role_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `user_id` (`user_id`,`role_id`,`tenant_id`), + KEY `tenant_id` (`tenant_id`), + KEY `role_id` (`role_id`) +) ENGINE=MyISAM AUTO_INCREMENT=8 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + +LOCK TABLES `user_roles` WRITE; +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1),(2,2,2,2),(3,2,2,3),(4,1,1,2),(5,1,1,NULL),(6,1,3,NULL),(7,1,4,NULL); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `password` varchar(255) DEFAULT NULL, + `email` varchar(255) DEFAULT NULL, + `enabled` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + KEY `tenant_id` (`tenant_id`) +) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + +LOCK TABLES `users` WRITE; +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES (1,'admin','secrete',NULL,1,NULL),(2,'demo','secrete',NULL,1,NULL); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-14 0:16:40 diff --git a/tests/legacy_d5.sqlite b/tests/legacy_d5.sqlite new file mode 100644 index 0000000000..d96dbf40fe --- /dev/null +++ b/tests/legacy_d5.sqlite @@ -0,0 +1,277 @@ +begin; +-- MySQL dump 10.13 Distrib 5.1.54, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.54-1ubuntu4 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `id` integer NOT NULL primary key autoincrement, + `user_id` integer NULL, + `tenant_id` integer NULL, + `type` varchar(20) NULL, + `key` varchar(255) NULL, + `secret` varchar(255) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + + +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +INSERT INTO `credentials` VALUES (1,1,1,'EC2','admin','secrete'); +INSERT INTO `credentials` VALUES (2,2,2,'EC2','demo','secrete'); +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `id` integer NOT NULL primary key autoincrement, + `region` varchar(255) NULL, + `service_id` integer NULL, + `public_url` varchar(2000) NULL, + `admin_url` varchar(2000) NULL, + `internal_url` varchar(2000) NULL, + `enabled` integer NULL, + `is_global` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + + +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne',1,'http://10.4.128.10:8774/v1.1/%tenant_id%','http://10.4.128.10:8774/v1.1/%tenant_id%','http://10.4.128.10:8774/v1.1/%tenant_id%',1,1); +INSERT INTO `endpoint_templates` VALUES (2,'RegionOne',2,'http://10.4.128.10:9292/v1.1/%tenant_id%','http://10.4.128.10:9292/v1.1/%tenant_id%','http://10.4.128.10:9292/v1.1/%tenant_id%',1,1); +INSERT INTO `endpoint_templates` VALUES (3,'RegionOne',3,'http://10.4.128.10:5000/v2.0','http://10.4.128.10:35357/v2.0','http://10.4.128.10:5000/v2.0',1,1); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `id` integer NOT NULL primary key autoincrement, + `tenant_id` integer NULL, + `endpoint_template_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + + +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; + + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `desc` varchar(255) NULL, + `service_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + + +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (1,'Admin',NULL,NULL); +INSERT INTO `roles` VALUES (2,'Member',NULL,NULL); +INSERT INTO `roles` VALUES (3,'KeystoneAdmin',NULL,NULL); +INSERT INTO `roles` VALUES (4,'KeystoneServiceAdmin',NULL,NULL); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; + + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `type` varchar(255) NULL, + `desc` varchar(255) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + + +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES (1,'nova','compute','Nova Compute Service'); +INSERT INTO `services` VALUES (2,'glance','image','Glance Image Service'); +INSERT INTO `services` VALUES (3,'keystone','identity','Keystone Identity Service'); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; + + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `desc` varchar(255) NULL, + `enabled` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + + +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (1,'admin',NULL,1); +INSERT INTO `tenants` VALUES (2,'demo',NULL,1); +INSERT INTO `tenants` VALUES (3,'invisible_to_admin',NULL,1); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; + + +-- +-- Table structure for table `token` +-- + +DROP TABLE IF EXISTS `token`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `token` ( + `id` varchar(255) NOT NULL, + `user_id` integer NULL, + `tenant_id` integer NULL, + `expires` datetime NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `token` +-- + + +/*!40000 ALTER TABLE `token` DISABLE KEYS */; +INSERT INTO `token` VALUES ('secrete',1,1,'2015-02-05 00:00:00'); +/*!40000 ALTER TABLE `token` ENABLE KEYS */; + + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `id` integer NOT NULL primary key autoincrement, + `user_id` integer NULL, + `role_id` integer NULL, + `tenant_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + + +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1); +INSERT INTO `user_roles` VALUES (2,2,2,2); +INSERT INTO `user_roles` VALUES (3,2,2,3); +INSERT INTO `user_roles` VALUES (4,1,1,2); +INSERT INTO `user_roles` VALUES (5,1,1,NULL); +INSERT INTO `user_roles` VALUES (6,1,3,NULL); +INSERT INTO `user_roles` VALUES (7,1,4,NULL); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; + + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `password` varchar(255) NULL, + `email` varchar(255) NULL, + `enabled` integer NULL, + `tenant_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + + +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES (1,'admin','secrete',NULL,1,NULL); +INSERT INTO `users` VALUES (2,'demo','secrete',NULL,1,NULL); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-14 0:16:40 +commit; diff --git a/tools/convert_to_sqlite.sh b/tools/convert_to_sqlite.sh new file mode 100755 index 0000000000..feb3202f25 --- /dev/null +++ b/tools/convert_to_sqlite.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +# ================================================================ +# +# Convert a mysql database dump into something sqlite3 understands. +# +# Adapted from +# http://stackoverflow.com/questions/489277/script-to-convert-mysql-dump-sql-file-into-format-that-can-be-imported-into-sqlit +# +# (c) 2010 Martin Czygan +# +# ================================================================ + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 " + exit +fi + +SRC=$1 +DST=$1.sqlite3.sql +DB=$1.sqlite3.db +ERR=$1.sqlite3.err + +cat $SRC | +grep -v ' KEY "' | +grep -v ' KEY `' | +grep -v ' UNIQUE KEY "' | +grep -v ' UNIQUE KEY `' | +grep -v ' PRIMARY KEY ' | + +sed 's/ENGINE=MyISAM/ /g' | +sed 's/DEFAULT/ /g' | +sed 's/CHARSET=[a-zA-Z0-9]*/ /g' | +sed 's/AUTO_INCREMENT=[0-9]*/ /g' | + +sed 's/\\r\\n/\\n/g' | +sed 's/\\"/"/g' | +sed '/^SET/d' | +sed 's/ unsigned / /g' | +sed 's/ auto_increment/ primary key autoincrement/g' | +sed 's/ AUTO_INCREMENT/ primary key autoincrement/g' | +sed 's/ smallint([0-9]*) / integer /g' | +sed 's/ tinyint([0-9]*) / integer /g' | +sed 's/ int([0-9]*) / integer /g' | +sed 's/ character set [^ ]* / /g' | +sed 's/ enum([^)]*) / varchar(255) /g' | +sed 's/ on update [^,]*//g' | +sed 's/UNLOCK TABLES;//g' | +sed 's/LOCK TABLES [^;]*;//g' | +perl -e 'local $/;$_=<>;s/,\n\)/\n\)/gs;print "begin;\n";print;print "commit;\n"' | +perl -pe ' + if (/^(INSERT.+?)\(/) { + $a=$1; + s/\\'\''/'\'\''/g; + s/\\n/\n/g; + s/\),\(/\);\n$a\(/g; + } + ' > $DST + +cat $DST | sqlite3 $DB > $ERR + +ERRORS=`cat $ERR | wc -l` + +if [ "$ERRORS" -eq "0" ]; then + echo "Conversion completed without error. Your db is ready under: $DB" + echo "\$ sqlite3 $DB" + rm -f $ERR +else + echo "There were errors during conversion. \ + Please review $ERR and $DST for details." +fi From aa2656c730d57515b0eba1f57c829b673490fdd5 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 17:31:58 -0800 Subject: [PATCH 326/334] add essex test as well Change-Id: Ib71a360568c6fdef4fdf8791d3f41b749391befb --- tests/legacy_essex.mysql | 309 +++++++++++++++++++++++++++++++++++ tests/legacy_essex.sqlite | 313 ++++++++++++++++++++++++++++++++++++ tests/test_import_legacy.py | 56 +++++++ 3 files changed, 678 insertions(+) create mode 100644 tests/legacy_essex.mysql create mode 100644 tests/legacy_essex.sqlite create mode 100644 tests/test_import_legacy.py diff --git a/tests/legacy_essex.mysql b/tests/legacy_essex.mysql new file mode 100644 index 0000000000..457ba7e984 --- /dev/null +++ b/tests/legacy_essex.mysql @@ -0,0 +1,309 @@ +-- MySQL dump 10.13 Distrib 5.1.58, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.58-1ubuntu1 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `user_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `secret` varchar(255) DEFAULT NULL, + `key` varchar(255) DEFAULT NULL, + `type` varchar(20) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + +LOCK TABLES `credentials` WRITE; +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `is_global` tinyint(1) DEFAULT NULL, + `region` varchar(255) DEFAULT NULL, + `public_url` varchar(2000) DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `internal_url` varchar(2000) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `service_id` int(11) DEFAULT NULL, + `admin_url` varchar(2000) DEFAULT NULL, + `version_id` varchar(20) DEFAULT NULL, + `version_list` varchar(2000) DEFAULT NULL, + `version_info` varchar(500) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM AUTO_INCREMENT=6 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + +LOCK TABLES `endpoint_templates` WRITE; +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:8774/v1.1/%tenant_id%',1,'http://4.2.2.1:8774/v1.1/%tenant_id%',1,1,'http://4.2.2.1:8774/v1.1/%tenant_id%',NULL,NULL,NULL),(1,'RegionOne','http://4.2.2.1:8773/services/Cloud',1,'http://4.2.2.1:8773/services/Cloud',2,2,'http://4.2.2.1:8773/services/Admin',NULL,NULL,NULL),(1,'RegionOne','http://4.2.2.1:9292/v1',1,'http://4.2.2.1:9292/v1',3,3,'http://4.2.2.1:9292/v1',NULL,NULL,NULL),(1,'RegionOne','http://4.2.2.1:5000/v2.0',1,'http://4.2.2.1:5000/v2.0',4,4,'http://4.2.2.1:35357/v2.0',NULL,NULL,NULL),(1,'RegionOne','http://4.2.2.1:8080/v1/AUTH_%tenant_id%',1,'http://4.2.2.1:8080/v1/AUTH_%tenant_id%',5,5,'http://4.2.2.1:8080/',NULL,NULL,NULL); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `endpoint_template_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`), + UNIQUE KEY `endpoint_template_id` (`endpoint_template_id`,`tenant_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + +LOCK TABLES `endpoints` WRITE; +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `migrate_version` +-- + +DROP TABLE IF EXISTS `migrate_version`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `migrate_version` ( + `repository_id` varchar(250) NOT NULL, + `repository_path` text, + `version` int(11) DEFAULT NULL, + PRIMARY KEY (`repository_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `migrate_version` +-- + +LOCK TABLES `migrate_version` WRITE; +/*!40000 ALTER TABLE `migrate_version` DISABLE KEYS */; +INSERT INTO `migrate_version` VALUES ('Keystone','/opt/stack/keystone/keystone/backends/sqlalchemy/migrate_repo',11); +/*!40000 ALTER TABLE `migrate_version` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `service_id` int(11) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`,`service_id`) +) ENGINE=MyISAM AUTO_INCREMENT=7 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + +LOCK TABLES `roles` WRITE; +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (NULL,NULL,1,'admin'),(NULL,NULL,2,'Member'),(NULL,NULL,3,'KeystoneAdmin'),(NULL,NULL,4,'KeystoneServiceAdmin'),(NULL,NULL,5,'sysadmin'),(NULL,NULL,6,'netadmin'); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `desc` varchar(255) DEFAULT NULL, + `type` varchar(255) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `owner_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + UNIQUE KEY `name_2` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=6 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + +LOCK TABLES `services` WRITE; +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES ('Nova Compute Service','compute',1,'nova',NULL),('EC2 Compatability Layer','ec2',2,'ec2',NULL),('Glance Image Service','image',3,'glance',NULL),('Keystone Identity Service','identity',4,'keystone',NULL),('Swift Service','object-store',5,'swift',NULL); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `desc` varchar(255) DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `uid` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `tenants_uid_key` (`uid`), + UNIQUE KEY `name` (`name`), + UNIQUE KEY `name_2` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + +LOCK TABLES `tenants` WRITE; +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (NULL,1,1,'admin','182c1fbf7eef44eda162ff3fd30c0a76'),(NULL,1,2,'demo','b1a7ea3a884f4d0685a98cd6e682a5da'),(NULL,1,3,'invisible_to_admin','f4d1eed9bb5d4d35a5f37af934f87574'); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `tokens` +-- + +DROP TABLE IF EXISTS `tokens`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tokens` ( + `tenant_id` int(11) DEFAULT NULL, + `expires` datetime DEFAULT NULL, + `user_id` int(11) DEFAULT NULL, + `id` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tokens` +-- + +LOCK TABLES `tokens` WRITE; +/*!40000 ALTER TABLE `tokens` DISABLE KEYS */; +INSERT INTO `tokens` VALUES (1,'2015-02-05 00:00:00',1,'123123123123123123123'); +/*!40000 ALTER TABLE `tokens` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `tenant_id` int(11) DEFAULT NULL, + `user_id` int(11) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `role_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `user_id` (`user_id`,`role_id`,`tenant_id`) +) ENGINE=MyISAM AUTO_INCREMENT=10 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + +LOCK TABLES `user_roles` WRITE; +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1),(2,2,2,2),(2,2,3,5),(2,2,4,6),(3,2,5,2),(2,1,6,1),(NULL,1,7,1),(NULL,1,8,3),(NULL,1,9,4); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `name` varchar(255) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `id` int(11) NOT NULL AUTO_INCREMENT, + `password` varchar(255) DEFAULT NULL, + `email` varchar(255) DEFAULT NULL, + `uid` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `users_uid_key` (`uid`), + UNIQUE KEY `name` (`name`), + UNIQUE KEY `name_2` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + +LOCK TABLES `users` WRITE; +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES ('admin',NULL,1,1,'$6$rounds=40000$hFXlgBSMi599197d$tmGKBpoGHNRsLB3ruK9f1wPvvtfWWuMEUzdqUAynsmmYXBK6eekyNHTzzhwXTM3mWpnaMHCI4mHPOycqmPJJc0',NULL,'c93b19ea3fa94484824213db8ac0afce'),('demo',NULL,1,2,'$6$rounds=40000$RBsX2ja9fdj2uTNQ$/wJOn510AYKW9BPFAJneVQAjm6TM0Ty11LG.u4.k4RhmoUcXNSjGKmQT6KO0SsvypMM7A.doWgt73V5rNnv5h.',NULL,'04c6697e88ff4667820903fcce05d904'); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-13 19:23:51 diff --git a/tests/legacy_essex.sqlite b/tests/legacy_essex.sqlite new file mode 100644 index 0000000000..33d0de20c8 --- /dev/null +++ b/tests/legacy_essex.sqlite @@ -0,0 +1,313 @@ +begin; +-- MySQL dump 10.13 Distrib 5.1.58, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.58-1ubuntu1 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `user_id` integer NULL, + `tenant_id` integer NULL, + `secret` varchar(255) NULL, + `key` varchar(255) NULL, + `type` varchar(20) NULL, + `id` integer NOT NULL primary key autoincrement +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + + +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `is_global` integer NULL, + `region` varchar(255) NULL, + `public_url` varchar(2000) NULL, + `enabled` integer NULL, + `internal_url` varchar(2000) NULL, + `id` integer NOT NULL primary key autoincrement, + `service_id` integer NULL, + `admin_url` varchar(2000) NULL, + `version_id` varchar(20) NULL, + `version_list` varchar(2000) NULL, + `version_info` varchar(500) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + + +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:8774/v1.1/%tenant_id%',1,'http://4.2.2.1:8774/v1.1/%tenant_id%',1,1,'http://4.2.2.1:8774/v1.1/%tenant_id%',NULL,NULL,NULL); +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:8773/services/Cloud',1,'http://4.2.2.1:8773/services/Cloud',2,2,'http://4.2.2.1:8773/services/Admin',NULL,NULL,NULL); +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:9292/v1',1,'http://4.2.2.1:9292/v1',3,3,'http://4.2.2.1:9292/v1',NULL,NULL,NULL); +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:5000/v2.0',1,'http://4.2.2.1:5000/v2.0',4,4,'http://4.2.2.1:35357/v2.0',NULL,NULL,NULL); +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne','http://4.2.2.1:8080/v1/AUTH_%tenant_id%',1,'http://4.2.2.1:8080/v1/AUTH_%tenant_id%',5,5,'http://4.2.2.1:8080/',NULL,NULL,NULL); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `endpoint_template_id` integer NULL, + `tenant_id` integer NULL, + `id` integer NOT NULL primary key autoincrement +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + + +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; + + +-- +-- Table structure for table `migrate_version` +-- + +DROP TABLE IF EXISTS `migrate_version`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `migrate_version` ( + `repository_id` varchar(250) NOT NULL, + `repository_path` text, + `version` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `migrate_version` +-- + + +/*!40000 ALTER TABLE `migrate_version` DISABLE KEYS */; +INSERT INTO `migrate_version` VALUES ('Keystone','/opt/stack/keystone/keystone/backends/sqlalchemy/migrate_repo',11); +/*!40000 ALTER TABLE `migrate_version` ENABLE KEYS */; + + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `service_id` integer NULL, + `desc` varchar(255) NULL, + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + + +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (NULL,NULL,1,'admin'); +INSERT INTO `roles` VALUES (NULL,NULL,2,'Member'); +INSERT INTO `roles` VALUES (NULL,NULL,3,'KeystoneAdmin'); +INSERT INTO `roles` VALUES (NULL,NULL,4,'KeystoneServiceAdmin'); +INSERT INTO `roles` VALUES (NULL,NULL,5,'sysadmin'); +INSERT INTO `roles` VALUES (NULL,NULL,6,'netadmin'); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; + + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `desc` varchar(255) NULL, + `type` varchar(255) NULL, + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `owner_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + + +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES ('Nova Compute Service','compute',1,'nova',NULL); +INSERT INTO `services` VALUES ('EC2 Compatability Layer','ec2',2,'ec2',NULL); +INSERT INTO `services` VALUES ('Glance Image Service','image',3,'glance',NULL); +INSERT INTO `services` VALUES ('Keystone Identity Service','identity',4,'keystone',NULL); +INSERT INTO `services` VALUES ('Swift Service','object-store',5,'swift',NULL); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; + + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `desc` varchar(255) NULL, + `enabled` integer NULL, + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `uid` varchar(255) NOT NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + + +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (NULL,1,1,'admin','182c1fbf7eef44eda162ff3fd30c0a76'); +INSERT INTO `tenants` VALUES (NULL,1,2,'demo','b1a7ea3a884f4d0685a98cd6e682a5da'); +INSERT INTO `tenants` VALUES (NULL,1,3,'invisible_to_admin','f4d1eed9bb5d4d35a5f37af934f87574'); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; + + +-- +-- Table structure for table `tokens` +-- + +DROP TABLE IF EXISTS `tokens`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tokens` ( + `tenant_id` integer NULL, + `expires` datetime NULL, + `user_id` integer NULL, + `id` varchar(255) NOT NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tokens` +-- + + +/*!40000 ALTER TABLE `tokens` DISABLE KEYS */; +INSERT INTO `tokens` VALUES (1,'2015-02-05 00:00:00',1,'123123123123123123123'); +/*!40000 ALTER TABLE `tokens` ENABLE KEYS */; + + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `tenant_id` integer NULL, + `user_id` integer NULL, + `id` integer NOT NULL primary key autoincrement, + `role_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + + +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1); +INSERT INTO `user_roles` VALUES (2,2,2,2); +INSERT INTO `user_roles` VALUES (2,2,3,5); +INSERT INTO `user_roles` VALUES (2,2,4,6); +INSERT INTO `user_roles` VALUES (3,2,5,2); +INSERT INTO `user_roles` VALUES (2,1,6,1); +INSERT INTO `user_roles` VALUES (NULL,1,7,1); +INSERT INTO `user_roles` VALUES (NULL,1,8,3); +INSERT INTO `user_roles` VALUES (NULL,1,9,4); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; + + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `name` varchar(255) NULL, + `tenant_id` integer NULL, + `enabled` integer NULL, + `id` integer NOT NULL primary key autoincrement, + `password` varchar(255) NULL, + `email` varchar(255) NULL, + `uid` varchar(255) NOT NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + + +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES ('admin',NULL,1,1,'$6$rounds=40000$hFXlgBSMi599197d$tmGKBpoGHNRsLB3ruK9f1wPvvtfWWuMEUzdqUAynsmmYXBK6eekyNHTzzhwXTM3mWpnaMHCI4mHPOycqmPJJc0',NULL,'c93b19ea3fa94484824213db8ac0afce'); +INSERT INTO `users` VALUES ('demo',NULL,1,2,'$6$rounds=40000$RBsX2ja9fdj2uTNQ$/wJOn510AYKW9BPFAJneVQAjm6TM0Ty11LG.u4.k4RhmoUcXNSjGKmQT6KO0SsvypMM7A.doWgt73V5rNnv5h.',NULL,'04c6697e88ff4667820903fcce05d904'); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-13 19:23:51 +commit; diff --git a/tests/test_import_legacy.py b/tests/test_import_legacy.py new file mode 100644 index 0000000000..f242724cdd --- /dev/null +++ b/tests/test_import_legacy.py @@ -0,0 +1,56 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import os + +import sqlite3 +#import sqlalchemy + +from keystone import config +from keystone import test +from keystone.common.sql import legacy +from keystone.common.sql import util as sql_util +from keystone.identity.backends import sql as identity_sql +from keystone.token.backends import sql as token_sql + + + +CONF = config.CONF + + +class ImportLegacy(test.TestCase): + def setUp(self): + super(ImportLegacy, self).setUp() + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf'), + test.testsdir('backend_sql.conf')]) + sql_util.setup_test_database() + self.identity_api = identity_sql.Identity() + + def setup_old_database(self, sql_dump): + sql_path = test.testsdir(sql_dump) + db_path = test.testsdir('%s.db' % sql_dump) + try: + os.unlink(db_path) + except OSError: + pass + script_str = open(sql_path).read().strip() + conn = sqlite3.connect(db_path) + conn.executescript(script_str) + conn.commit() + return db_path + + def test_import_d5(self): + db_path = self.setup_old_database('legacy_d5.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() + + user_ref = self.identity_api.get_user('1') + self.assertEquals(user_ref['name'], 'admin') + + def test_import_essex(self): + db_path = self.setup_old_database('legacy_essex.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() + + user_ref = self.identity_api.get_user('c93b19ea3fa94484824213db8ac0afce') + self.assertEquals(user_ref['name'], 'admin') From 48f2f650c8b622b55e67610081336055ec9a2c8e Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 16:49:54 -0800 Subject: [PATCH 327/334] change password hash Change-Id: Idd5d09dc114cbb0cbd63e23e4178bb74d081c789 --- keystone/common/utils.py | 13 ++++++------- tests/test_overrides.conf | 2 +- tools/pip-requires | 2 +- tools/pip-requires-test | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 69fb5edab6..9c52b93f4f 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -25,14 +25,14 @@ import subprocess import sys import urllib -import bcrypt +import passlib.hash from keystone import config from keystone.common import logging CONF = config.CONF -config.register_int('bcrypt_strength', default=12) +config.register_int('crypt_strength', default=40000) def import_class(import_str): @@ -143,9 +143,9 @@ class Ec2Signer(object): def hash_password(password): """Hash a password. Hard.""" - salt = bcrypt.gensalt(CONF.bcrypt_strength) - password_utf8 = password.encode('utf-8') - return bcrypt.hashpw(password_utf8, salt) + h = passlib.hash.sha512_crypt.encrypt(password.encode('utf-8'), + rounds=CONF.crypt_strength) + return h def check_password(password, hashed): @@ -158,8 +158,7 @@ def check_password(password, hashed): if password is None: return False password_utf8 = password.encode('utf-8') - check = bcrypt.hashpw(password_utf8, hashed) - return check == hashed + return passlib.hash.sha512_crypt.verify(password_utf8, hashed) # From python 2.7 diff --git a/tests/test_overrides.conf b/tests/test_overrides.conf index 399110d7e8..501293c6e7 100644 --- a/tests/test_overrides.conf +++ b/tests/test_overrides.conf @@ -1,5 +1,5 @@ [DEFAULT] -bcrypt_strength = 2 +crypt_strength = 10 [catalog] template_file = default_catalog.templates diff --git a/tools/pip-requires b/tools/pip-requires index 03230ae854..1ff098cd54 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -8,7 +8,7 @@ routes pyCLI sqlalchemy sqlalchemy-migrate -py-bcrypt +passlib # for python-novaclient prettytable diff --git a/tools/pip-requires-test b/tools/pip-requires-test index f16a3b927d..873712ecd3 100644 --- a/tools/pip-requires-test +++ b/tools/pip-requires-test @@ -8,7 +8,7 @@ routes pyCLI sqlalchemy sqlalchemy-migrate -py-bcrypt +passlib python-memcached # keystonelight testing dependencies From b4096290d149a04d5f3691025ad4adbfb4f4d4eb Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 17:38:26 -0800 Subject: [PATCH 328/334] add legacy diablo import tests Change-Id: Id96b376ad92d906d14c3183eec2739fa34a2e51b --- tests/legacy_diablo.mysql | 281 +++++++++++++++++++++++++++++++++++ tests/legacy_diablo.sqlite | 283 ++++++++++++++++++++++++++++++++++++ tests/test_import_legacy.py | 8 + 3 files changed, 572 insertions(+) create mode 100644 tests/legacy_diablo.mysql create mode 100644 tests/legacy_diablo.sqlite diff --git a/tests/legacy_diablo.mysql b/tests/legacy_diablo.mysql new file mode 100644 index 0000000000..543f439f8a --- /dev/null +++ b/tests/legacy_diablo.mysql @@ -0,0 +1,281 @@ +-- MySQL dump 10.13 Distrib 5.1.58, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.58-1ubuntu1 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `user_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `type` varchar(20) DEFAULT NULL, + `key` varchar(255) DEFAULT NULL, + `secret` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `tenant_id` (`tenant_id`), + KEY `user_id` (`user_id`) +) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + +LOCK TABLES `credentials` WRITE; +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +INSERT INTO `credentials` VALUES (1,1,1,'EC2','admin','secrete'),(2,2,2,'EC2','demo','secrete'); +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `region` varchar(255) DEFAULT NULL, + `service_id` int(11) DEFAULT NULL, + `public_url` varchar(2000) DEFAULT NULL, + `admin_url` varchar(2000) DEFAULT NULL, + `internal_url` varchar(2000) DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `is_global` tinyint(1) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `service_id` (`service_id`) +) ENGINE=MyISAM AUTO_INCREMENT=5 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + +LOCK TABLES `endpoint_templates` WRITE; +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne',1,'http://192.168.2.10:8774/v1.1/%tenant_id%','http://192.168.2.10:8774/v1.1/%tenant_id%','http://192.168.2.10:8774/v1.1/%tenant_id%',1,1),(2,'RegionOne',2,'http://192.168.2.10:9292/v1','http://192.168.2.10:9292/v1','http://192.168.2.10:9292/v1',1,1),(3,'RegionOne',3,'http://192.168.2.10:5000/v2.0','http://192.168.2.10:35357/v2.0','http://192.168.2.10:5000/v2.0',1,1),(4,'RegionOne',4,'http://192.168.2.10:8080/v1/AUTH_%tenant_id%','http://192.168.2.10:8080/','http://192.168.2.10:8080/v1/AUTH_%tenant_id%',1,1); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tenant_id` int(11) DEFAULT NULL, + `endpoint_template_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `endpoint_template_id` (`endpoint_template_id`,`tenant_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + +LOCK TABLES `endpoints` WRITE; +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + `service_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`,`service_id`), + KEY `service_id` (`service_id`) +) ENGINE=MyISAM AUTO_INCREMENT=7 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + +LOCK TABLES `roles` WRITE; +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (1,'Admin',NULL,NULL),(2,'Member',NULL,NULL),(3,'KeystoneAdmin',NULL,NULL),(4,'KeystoneServiceAdmin',NULL,NULL),(5,'sysadmin',NULL,NULL),(6,'netadmin',NULL,NULL); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `type` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=5 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + +LOCK TABLES `services` WRITE; +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES (1,'nova','compute','Nova Compute Service'),(2,'glance','image','Glance Image Service'),(3,'keystone','identity','Keystone Identity Service'),(4,'swift','object-store','Swift Service'); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `desc` varchar(255) DEFAULT NULL, + `enabled` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + +LOCK TABLES `tenants` WRITE; +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (1,'admin',NULL,1),(2,'demo',NULL,1),(3,'invisible_to_admin',NULL,1); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `token` +-- + +DROP TABLE IF EXISTS `token`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `token` ( + `id` varchar(255) NOT NULL, + `user_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + `expires` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `token` +-- + +LOCK TABLES `token` WRITE; +/*!40000 ALTER TABLE `token` DISABLE KEYS */; +INSERT INTO `token` VALUES ('secrete',1,1,'2015-02-05 00:00:00'); +/*!40000 ALTER TABLE `token` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `user_id` int(11) DEFAULT NULL, + `role_id` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `user_id` (`user_id`,`role_id`,`tenant_id`), + KEY `tenant_id` (`tenant_id`), + KEY `role_id` (`role_id`) +) ENGINE=MyISAM AUTO_INCREMENT=10 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + +LOCK TABLES `user_roles` WRITE; +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1),(2,2,2,2),(3,2,5,2),(4,2,6,2),(5,2,2,3),(6,1,1,2),(7,1,1,NULL),(8,1,3,NULL),(9,1,4,NULL); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + `password` varchar(255) DEFAULT NULL, + `email` varchar(255) DEFAULT NULL, + `enabled` int(11) DEFAULT NULL, + `tenant_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + KEY `tenant_id` (`tenant_id`) +) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + +LOCK TABLES `users` WRITE; +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES (1,'admin','secrete',NULL,1,NULL),(2,'demo','secrete',NULL,1,NULL); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-13 17:30:03 diff --git a/tests/legacy_diablo.sqlite b/tests/legacy_diablo.sqlite new file mode 100644 index 0000000000..edf15be4c7 --- /dev/null +++ b/tests/legacy_diablo.sqlite @@ -0,0 +1,283 @@ +begin; +-- MySQL dump 10.13 Distrib 5.1.58, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: keystone +-- ------------------------------------------------------ +-- Server version 5.1.58-1ubuntu1 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `credentials` +-- + +DROP TABLE IF EXISTS `credentials`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `credentials` ( + `id` integer NOT NULL primary key autoincrement, + `user_id` integer NULL, + `tenant_id` integer NULL, + `type` varchar(20) NULL, + `key` varchar(255) NULL, + `secret` varchar(255) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `credentials` +-- + + +/*!40000 ALTER TABLE `credentials` DISABLE KEYS */; +INSERT INTO `credentials` VALUES (1,1,1,'EC2','admin','secrete'); +INSERT INTO `credentials` VALUES (2,2,2,'EC2','demo','secrete'); +/*!40000 ALTER TABLE `credentials` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoint_templates` +-- + +DROP TABLE IF EXISTS `endpoint_templates`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoint_templates` ( + `id` integer NOT NULL primary key autoincrement, + `region` varchar(255) NULL, + `service_id` integer NULL, + `public_url` varchar(2000) NULL, + `admin_url` varchar(2000) NULL, + `internal_url` varchar(2000) NULL, + `enabled` integer NULL, + `is_global` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoint_templates` +-- + + +/*!40000 ALTER TABLE `endpoint_templates` DISABLE KEYS */; +INSERT INTO `endpoint_templates` VALUES (1,'RegionOne',1,'http://192.168.2.10:8774/v1.1/%tenant_id%','http://192.168.2.10:8774/v1.1/%tenant_id%','http://192.168.2.10:8774/v1.1/%tenant_id%',1,1); +INSERT INTO `endpoint_templates` VALUES (2,'RegionOne',2,'http://192.168.2.10:9292/v1','http://192.168.2.10:9292/v1','http://192.168.2.10:9292/v1',1,1); +INSERT INTO `endpoint_templates` VALUES (3,'RegionOne',3,'http://192.168.2.10:5000/v2.0','http://192.168.2.10:35357/v2.0','http://192.168.2.10:5000/v2.0',1,1); +INSERT INTO `endpoint_templates` VALUES (4,'RegionOne',4,'http://192.168.2.10:8080/v1/AUTH_%tenant_id%','http://192.168.2.10:8080/','http://192.168.2.10:8080/v1/AUTH_%tenant_id%',1,1); +/*!40000 ALTER TABLE `endpoint_templates` ENABLE KEYS */; + + +-- +-- Table structure for table `endpoints` +-- + +DROP TABLE IF EXISTS `endpoints`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `endpoints` ( + `id` integer NOT NULL primary key autoincrement, + `tenant_id` integer NULL, + `endpoint_template_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `endpoints` +-- + + +/*!40000 ALTER TABLE `endpoints` DISABLE KEYS */; +/*!40000 ALTER TABLE `endpoints` ENABLE KEYS */; + + +-- +-- Table structure for table `roles` +-- + +DROP TABLE IF EXISTS `roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `roles` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `desc` varchar(255) NULL, + `service_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `roles` +-- + + +/*!40000 ALTER TABLE `roles` DISABLE KEYS */; +INSERT INTO `roles` VALUES (1,'Admin',NULL,NULL); +INSERT INTO `roles` VALUES (2,'Member',NULL,NULL); +INSERT INTO `roles` VALUES (3,'KeystoneAdmin',NULL,NULL); +INSERT INTO `roles` VALUES (4,'KeystoneServiceAdmin',NULL,NULL); +INSERT INTO `roles` VALUES (5,'sysadmin',NULL,NULL); +INSERT INTO `roles` VALUES (6,'netadmin',NULL,NULL); +/*!40000 ALTER TABLE `roles` ENABLE KEYS */; + + +-- +-- Table structure for table `services` +-- + +DROP TABLE IF EXISTS `services`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `services` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `type` varchar(255) NULL, + `desc` varchar(255) NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `services` +-- + + +/*!40000 ALTER TABLE `services` DISABLE KEYS */; +INSERT INTO `services` VALUES (1,'nova','compute','Nova Compute Service'); +INSERT INTO `services` VALUES (2,'glance','image','Glance Image Service'); +INSERT INTO `services` VALUES (3,'keystone','identity','Keystone Identity Service'); +INSERT INTO `services` VALUES (4,'swift','object-store','Swift Service'); +/*!40000 ALTER TABLE `services` ENABLE KEYS */; + + +-- +-- Table structure for table `tenants` +-- + +DROP TABLE IF EXISTS `tenants`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `tenants` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `desc` varchar(255) NULL, + `enabled` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `tenants` +-- + + +/*!40000 ALTER TABLE `tenants` DISABLE KEYS */; +INSERT INTO `tenants` VALUES (1,'admin',NULL,1); +INSERT INTO `tenants` VALUES (2,'demo',NULL,1); +INSERT INTO `tenants` VALUES (3,'invisible_to_admin',NULL,1); +/*!40000 ALTER TABLE `tenants` ENABLE KEYS */; + + +-- +-- Table structure for table `token` +-- + +DROP TABLE IF EXISTS `token`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `token` ( + `id` varchar(255) NOT NULL, + `user_id` integer NULL, + `tenant_id` integer NULL, + `expires` datetime NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `token` +-- + + +/*!40000 ALTER TABLE `token` DISABLE KEYS */; +INSERT INTO `token` VALUES ('secrete',1,1,'2015-02-05 00:00:00'); +/*!40000 ALTER TABLE `token` ENABLE KEYS */; + + +-- +-- Table structure for table `user_roles` +-- + +DROP TABLE IF EXISTS `user_roles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user_roles` ( + `id` integer NOT NULL primary key autoincrement, + `user_id` integer NULL, + `role_id` integer NULL, + `tenant_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user_roles` +-- + + +/*!40000 ALTER TABLE `user_roles` DISABLE KEYS */; +INSERT INTO `user_roles` VALUES (1,1,1,1); +INSERT INTO `user_roles` VALUES (2,2,2,2); +INSERT INTO `user_roles` VALUES (3,2,5,2); +INSERT INTO `user_roles` VALUES (4,2,6,2); +INSERT INTO `user_roles` VALUES (5,2,2,3); +INSERT INTO `user_roles` VALUES (6,1,1,2); +INSERT INTO `user_roles` VALUES (7,1,1,NULL); +INSERT INTO `user_roles` VALUES (8,1,3,NULL); +INSERT INTO `user_roles` VALUES (9,1,4,NULL); +/*!40000 ALTER TABLE `user_roles` ENABLE KEYS */; + + +-- +-- Table structure for table `users` +-- + +DROP TABLE IF EXISTS `users`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `users` ( + `id` integer NOT NULL primary key autoincrement, + `name` varchar(255) NULL, + `password` varchar(255) NULL, + `email` varchar(255) NULL, + `enabled` integer NULL, + `tenant_id` integer NULL +) ; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `users` +-- + + +/*!40000 ALTER TABLE `users` DISABLE KEYS */; +INSERT INTO `users` VALUES (1,'admin','secrete',NULL,1,NULL); +INSERT INTO `users` VALUES (2,'demo','secrete',NULL,1,NULL); +/*!40000 ALTER TABLE `users` ENABLE KEYS */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-02-13 17:30:03 +commit; diff --git a/tests/test_import_legacy.py b/tests/test_import_legacy.py index f242724cdd..0551798541 100644 --- a/tests/test_import_legacy.py +++ b/tests/test_import_legacy.py @@ -47,6 +47,14 @@ class ImportLegacy(test.TestCase): user_ref = self.identity_api.get_user('1') self.assertEquals(user_ref['name'], 'admin') + def test_import_diablo(self): + db_path = self.setup_old_database('legacy_diablo.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() + + user_ref = self.identity_api.get_user('1') + self.assertEquals(user_ref['name'], 'admin') + def test_import_essex(self): db_path = self.setup_old_database('legacy_essex.sqlite') migration = legacy.LegacyMigration('sqlite:///%s' % db_path) From ed793ad5365e33e2fda54c3900c1ad9b2c93dc37 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 20:34:46 -0800 Subject: [PATCH 329/334] make sure passwords work after migration Change-Id: I0086a362d772bf158e3fdc12fb42c1c7c50d50dd --- keystone/common/utils.py | 5 ++++- tests/test_import_legacy.py | 21 ++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/keystone/common/utils.py b/keystone/common/utils.py index 9c52b93f4f..b5269bed2f 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -143,7 +143,10 @@ class Ec2Signer(object): def hash_password(password): """Hash a password. Hard.""" - h = passlib.hash.sha512_crypt.encrypt(password.encode('utf-8'), + password_utf8 = password.encode('utf-8') + if passlib.hash.sha512_crypt.identify(password_utf8): + return password_utf8 + h = passlib.hash.sha512_crypt.encrypt(password_utf8, rounds=CONF.crypt_strength) return h diff --git a/tests/test_import_legacy.py b/tests/test_import_legacy.py index 0551798541..2db34cb881 100644 --- a/tests/test_import_legacy.py +++ b/tests/test_import_legacy.py @@ -44,21 +44,36 @@ class ImportLegacy(test.TestCase): migration = legacy.LegacyMigration('sqlite:///%s' % db_path) migration.migrate_all() - user_ref = self.identity_api.get_user('1') + admin_id = '1' + user_ref = self.identity_api.get_user(admin_id) self.assertEquals(user_ref['name'], 'admin') + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') + def test_import_diablo(self): db_path = self.setup_old_database('legacy_diablo.sqlite') migration = legacy.LegacyMigration('sqlite:///%s' % db_path) migration.migrate_all() - user_ref = self.identity_api.get_user('1') + admin_id = '1' + user_ref = self.identity_api.get_user(admin_id) self.assertEquals(user_ref['name'], 'admin') + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') + def test_import_essex(self): db_path = self.setup_old_database('legacy_essex.sqlite') migration = legacy.LegacyMigration('sqlite:///%s' % db_path) migration.migrate_all() - user_ref = self.identity_api.get_user('c93b19ea3fa94484824213db8ac0afce') + admin_id = 'c93b19ea3fa94484824213db8ac0afce' + user_ref = self.identity_api.get_user(admin_id) self.assertEquals(user_ref['name'], 'admin') + + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') From e1a9a1f06f83f9795e03d726a9e41db6dd9b9a9f Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 13 Feb 2012 14:15:33 -0800 Subject: [PATCH 330/334] Handle unicode keys in memcache token backend * Cast keys to str in memcache backend * Emulate encoding error in fake memcache client * Fixes bug 931746 Change-Id: I13bc573d4aca6849b1b8128ab55823545d5a3a11 --- keystone/token/backends/memcache.py | 2 +- tests/test_backend_memcache.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/keystone/token/backends/memcache.py b/keystone/token/backends/memcache.py index d4674ce9e5..8dfb373197 100644 --- a/keystone/token/backends/memcache.py +++ b/keystone/token/backends/memcache.py @@ -27,7 +27,7 @@ class Token(token.Driver): return self._memcache_client def _prefix_token_id(self, token_id): - return 'token-%s' % token_id + return 'token-%s' % token_id.encode('utf-8') def get_token(self, token_id): ptk = self._prefix_token_id(token_id) diff --git a/tests/test_backend_memcache.py b/tests/test_backend_memcache.py index f665fb4bb5..b320b1f91a 100644 --- a/tests/test_backend_memcache.py +++ b/tests/test_backend_memcache.py @@ -1,5 +1,9 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import uuid + +import memcache + from keystone import exception from keystone import test from keystone.token.backends import memcache as token_memcache @@ -14,8 +18,13 @@ class MemcacheClient(object): """Ignores the passed in args.""" self.cache = {} + def check_key(self, key): + if not isinstance(key, str): + raise memcache.Client.MemcachedStringEncodingError() + def get(self, key): """Retrieves the value for a key or None.""" + self.check_key(key) try: return self.cache[key] except KeyError: @@ -23,10 +32,12 @@ class MemcacheClient(object): def set(self, key, value): """Sets the value for a key.""" + self.check_key(key) self.cache[key] = value return True def delete(self, key): + self.check_key(key) try: del self.cache[key] except KeyError: @@ -39,3 +50,9 @@ class MemcacheToken(test.TestCase, test_backend.TokenTests): super(MemcacheToken, self).setUp() fake_client = MemcacheClient() self.token_api = token_memcache.Token(client=fake_client) + + def test_get_unicode(self): + token_id = unicode(uuid.uuid4().hex) + data = {'id': token_id, 'a': 'b'} + self.token_api.create_token(token_id, data) + self.token_api.get_token(token_id) From 27db5cbc05864b8c130eded9082ee82f7e722c34 Mon Sep 17 00:00:00 2001 From: termie Date: Mon, 13 Feb 2012 22:07:10 -0800 Subject: [PATCH 331/334] add catalog export Change-Id: I66a7b3e8136757979c96984242b2bbd5f390b9a0 --- keystone/catalog/backends/templated.py | 52 ++++++----- keystone/cli.py | 17 ++++ keystone/common/sql/legacy.py | 32 ++++++- tests/test_import_legacy.py | 118 +++++++++++++++---------- 4 files changed, 144 insertions(+), 75 deletions(-) diff --git a/keystone/catalog/backends/templated.py b/keystone/catalog/backends/templated.py index e8290212ae..b6e7036fdd 100644 --- a/keystone/catalog/backends/templated.py +++ b/keystone/catalog/backends/templated.py @@ -9,6 +9,33 @@ CONF = config.CONF config.register_str('template_file', group='catalog') +def parse_templates(template_lines): + o = {} + for line in template_lines: + if ' = ' not in line: + continue + + k, v = line.strip().split(' = ') + if not k.startswith('catalog.'): + continue + + parts = k.split('.') + + region = parts[1] + # NOTE(termie): object-store insists on having a dash + service = parts[2].replace('_', '-') + key = parts[3] + + region_ref = o.get(region, {}) + service_ref = region_ref.get(service, {}) + service_ref[key] = v + + region_ref[service] = service_ref + o[region] = region_ref + + return o + + class TemplatedCatalog(kvs.Catalog): """A backend that generates endpoints for the Catalog based on templates. @@ -49,30 +76,7 @@ class TemplatedCatalog(kvs.Catalog): super(TemplatedCatalog, self).__init__() def _load_templates(self, template_file): - o = {} - for line in open(template_file): - if ' = ' not in line: - continue - - k, v = line.strip().split(' = ') - if not k.startswith('catalog.'): - continue - - parts = k.split('.') - - region = parts[1] - # NOTE(termie): object-store insists on having a dash - service = parts[2].replace('_', '-') - key = parts[3] - - region_ref = o.get(region, {}) - service_ref = region_ref.get(service, {}) - service_ref[key] = v - - region_ref[service] = service_ref - o[region] = region_ref - - self.templates = o + self.templates = parse_templates(open(template_file)) def get_catalog(self, user_id, tenant_id, metadata=None): d = dict(CONF.iteritems()) diff --git a/keystone/cli.py b/keystone/cli.py index b7e7dfbbff..b6b5abb8f3 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -74,8 +74,25 @@ class ImportLegacy(BaseApp): migration.migrate_all() +class ExportLegacyCatalog(BaseApp): + """Export the service catalog from a legacy database.""" + + name = 'export_legacy_catalog' + + def __init__(self, *args, **kw): + super(ExportLegacyCatalog, self).__init__(*args, **kw) + self.add_param('old_db', nargs=1) + + def main(self): + from keystone.common.sql import legacy + old_db = self.params.old_db[0] + migration = legacy.LegacyMigration(old_db) + print '\n'.join(migration.dump_catalog()) + + CMDS = {'db_sync': DbSync, 'import_legacy': ImportLegacy, + 'export_legacy_catalog': ExportLegacyCatalog, } diff --git a/keystone/common/sql/legacy.py b/keystone/common/sql/legacy.py index 1d6c7d6151..d8bbb2c5f8 100644 --- a/keystone/common/sql/legacy.py +++ b/keystone/common/sql/legacy.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import re + import sqlalchemy from keystone.identity.backends import sql as identity_sql @@ -23,6 +25,12 @@ def export_db(db): return migration_data +def _translate_replacements(s): + if '%' not in str(s): + return s + return re.sub(r'%([\w_]+)%', r'$(\1)s', s) + + class LegacyMigration(object): def __init__(self, db_string): self.db = sqlalchemy.create_engine(db_string) @@ -41,9 +49,29 @@ class LegacyMigration(object): self._migrate_user_roles() self._migrate_tokens() - def dump_catalog(self, path): + def dump_catalog(self): """Generate the contents of a catalog templates file.""" - pass + self._export_legacy_db() + + services_by_id = dict((x['id'], x) for x in self._data['services']) + template = 'catalog.%(region)s.%(service_type)s.%(key)s = %(value)s' + + o = [] + for row in self._data['endpoint_templates']: + service = services_by_id[row['service_id']] + d = {'service_type': service['type'], + 'region': row['region']} + + for x in ['internal_url', 'public_url', 'admin_url', 'enabled']: + d['key'] = x.replace('_u', 'U') + d['value'] = _translate_replacements(row[x]) + o.append(template % d) + + d['key'] = 'name' + d['value'] = service['desc'] + o.append(template % d) + + return o def _export_legacy_db(self): self._data = export_db(self.db) diff --git a/tests/test_import_legacy.py b/tests/test_import_legacy.py index 2db34cb881..133ebcda43 100644 --- a/tests/test_import_legacy.py +++ b/tests/test_import_legacy.py @@ -11,6 +11,7 @@ from keystone.common.sql import legacy from keystone.common.sql import util as sql_util from keystone.identity.backends import sql as identity_sql from keystone.token.backends import sql as token_sql +from keystone.catalog.backends import templated as catalog_templated @@ -18,62 +19,81 @@ CONF = config.CONF class ImportLegacy(test.TestCase): - def setUp(self): - super(ImportLegacy, self).setUp() - CONF(config_files=[test.etcdir('keystone.conf'), - test.testsdir('test_overrides.conf'), - test.testsdir('backend_sql.conf')]) - sql_util.setup_test_database() - self.identity_api = identity_sql.Identity() + def setUp(self): + super(ImportLegacy, self).setUp() + CONF(config_files=[test.etcdir('keystone.conf'), + test.testsdir('test_overrides.conf'), + test.testsdir('backend_sql.conf')]) + sql_util.setup_test_database() + self.identity_api = identity_sql.Identity() - def setup_old_database(self, sql_dump): - sql_path = test.testsdir(sql_dump) - db_path = test.testsdir('%s.db' % sql_dump) - try: - os.unlink(db_path) - except OSError: - pass - script_str = open(sql_path).read().strip() - conn = sqlite3.connect(db_path) - conn.executescript(script_str) - conn.commit() - return db_path + def setup_old_database(self, sql_dump): + sql_path = test.testsdir(sql_dump) + db_path = test.testsdir('%s.db' % sql_dump) + try: + os.unlink(db_path) + except OSError: + pass + script_str = open(sql_path).read().strip() + conn = sqlite3.connect(db_path) + conn.executescript(script_str) + conn.commit() + return db_path - def test_import_d5(self): - db_path = self.setup_old_database('legacy_d5.sqlite') - migration = legacy.LegacyMigration('sqlite:///%s' % db_path) - migration.migrate_all() + def test_import_d5(self): + db_path = self.setup_old_database('legacy_d5.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() - admin_id = '1' - user_ref = self.identity_api.get_user(admin_id) - self.assertEquals(user_ref['name'], 'admin') + admin_id = '1' + user_ref = self.identity_api.get_user(admin_id) + self.assertEquals(user_ref['name'], 'admin') + self.assertEquals(user_ref['enabled'], True) - # check password hashing - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - user_id=admin_id, password='secrete') + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') - def test_import_diablo(self): - db_path = self.setup_old_database('legacy_diablo.sqlite') - migration = legacy.LegacyMigration('sqlite:///%s' % db_path) - migration.migrate_all() + # check catalog + self._check_catalog(migration) - admin_id = '1' - user_ref = self.identity_api.get_user(admin_id) - self.assertEquals(user_ref['name'], 'admin') + def test_import_diablo(self): + db_path = self.setup_old_database('legacy_diablo.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() - # check password hashing - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - user_id=admin_id, password='secrete') + admin_id = '1' + user_ref = self.identity_api.get_user(admin_id) + self.assertEquals(user_ref['name'], 'admin') + self.assertEquals(user_ref['enabled'], True) - def test_import_essex(self): - db_path = self.setup_old_database('legacy_essex.sqlite') - migration = legacy.LegacyMigration('sqlite:///%s' % db_path) - migration.migrate_all() + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') - admin_id = 'c93b19ea3fa94484824213db8ac0afce' - user_ref = self.identity_api.get_user(admin_id) - self.assertEquals(user_ref['name'], 'admin') + # check catalog + self._check_catalog(migration) - # check password hashing - user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( - user_id=admin_id, password='secrete') + def test_import_essex(self): + db_path = self.setup_old_database('legacy_essex.sqlite') + migration = legacy.LegacyMigration('sqlite:///%s' % db_path) + migration.migrate_all() + + admin_id = 'c93b19ea3fa94484824213db8ac0afce' + user_ref = self.identity_api.get_user(admin_id) + self.assertEquals(user_ref['name'], 'admin') + self.assertEquals(user_ref['enabled'], True) + + # check password hashing + user_ref, tenant_ref, metadata_ref = self.identity_api.authenticate( + user_id=admin_id, password='secrete') + + # check catalog + self._check_catalog(migration) + + def _check_catalog(self, migration): + catalog_lines = migration.dump_catalog() + catalog = catalog_templated.parse_templates(catalog_lines) + self.assert_('RegionOne' in catalog) + self.assert_('compute' in catalog['RegionOne']) + self.assert_('adminUrl' in catalog['RegionOne']['compute']) From 448c6414a176831b400ed5a1618fe89a8780968b Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Tue, 14 Feb 2012 12:30:23 -0800 Subject: [PATCH 332/334] Update docs to for current keystone-manage usage * Document how to manage users, roles, tenants, services, etc with keystoneclient cli * keystone-manage only does db_sync right now * Fixes bug 931837 * Add docs for import_legacy and export_legacy_catalog Change-Id: I7f55fd607363d0cd4f1646564e430dfb5b12855f --- docs/source/configuration.rst | 370 ++++++++++++++-------------- docs/source/configuringservices.rst | 30 +-- docs/source/man/keystone-manage.rst | 156 +----------- 3 files changed, 207 insertions(+), 349 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 586b66bc8e..7b975b1082 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -94,130 +94,152 @@ primary/public API interface). Both of these run in a single process. Initializing Keystone ===================== -Keystone must be running in order to initialize data within it. This is -because the keystone-manage commands are all used the same REST API that other -OpenStack systems utilize. - -General keystone-manage options: --------------------------------- - -* ``--id-only`` : causes ``keystone-manage`` to return only the UUID result - from the API call. - -* ``--endpoint`` : allows you to specify the keystone endpoint to communicate - with. The default endpoint is http://localhost:35357/v2.0' - -* ``--auth-token`` : provides the authorization token - -``keystone-manage`` is set up to expect commands in the general form of -``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to -provide additional information to the command. For example, the command -``tenant`` has the subcommand ``create``, which takes the required keyword -``tenant_name``:: - - keystone-manage tenant create tenant_name=example_tenant - -Invoking keystone-manage by itself will give you some usage information. - -Available keystone-manage commands: +keystone-manage is designed to execute commands that cannot be administered +through the normal REST api. At the moment, the following calls are supported: * ``db_sync``: Sync the database. -* ``ec2``: no docs -* ``role``: Role CRUD functions. -* ``service``: Service CRUD functions. -* ``tenant``: Tenant CRUD functions. -* ``token``: Token CRUD functions. -* ``user``: User CRUD functions. +* ``import_legacy``: Import a legacy (pre-essex) version of the db. +* ``export_legacy_catalog``: Export service catalog from a legacy (pre-essex) db. + + +Generally, the following is the first step after a source installation:: + + keystone-manage db_sync + +Invoking keystone-manage by itself will give you additional usage information. + +Adding Users, Tenants, and Roles with python-keystoneclient +=========================================================== + +User, tenants, and roles must be administered using admin credentials. +There are two ways to configure python-keystoneclient to use admin +credentials, using the token auth method, or password auth method. + +Token Auth Method +----------------- +To use keystone client using token auth, set the following flags + +* ``--endpoint SERVIVE_ENDPOINT`` : allows you to specify the keystone endpoint to communicate + with. The default endpoint is http://localhost:35357/v2.0' +* ``--token SERVIVE_TOKEN`` : your administrator service token. + +Password Auth Method +-------------------- + +* ``--username OS_USERNAME`` : allows you to specify the keystone endpoint to communicate + with. For example, http://localhost:35357/v2.0' +* ``--password OS_PASSWORD`` : Your administrator password +* ``--tenant_name OS_TENANT_NAME`` : Name of your tenant +* ``--auth_url OS_AUTH_URL`` : url of your keystone auth server, for example +http://localhost:5000/v2.0' + +Example usage +------------- +``keystone`` is set up to expect commands in the general form of +``keystone`` ``command`` ``argument``, followed by flag-like keyword arguments to +provide additional (often optional) information. For example, the command +``user-list`` and ``tenant-create`` can be invoked as follows:: + + # Using token auth env variables + export SERVICE_ENDPOINT=http://127.0.0.1:5000/v2.0/ + export SERVICE_TOKEN=secrete_token + keystone user-list + keystone tenant-create --name=demo + + # Using token auth flags + keystone --token=secrete --endpoint=http://127.0.0.1:5000/v2.0/ user-list + keystone --token=secrete --endpoint=http://127.0.0.1:5000/v2.0/ tenant-create --name=demo + + # Using user + password + tenant_name env variables + export OS_USERNAME=admin + export OS_PASSWORD=secrete + export OS_TENANT_NAME=admin + keystone user-list + keystone tenant-create --name=demo + + # Using user + password + tenant_name flags + keystone --username=admin --password=secrete --tenant_name=admin user-list + keystone --username=admin --password=secrete --tenant_name=admin tenant-create --name=demo Tenants ------- Tenants are the high level grouping within Keystone that represent groups of users. A tenant is the grouping that owns virtual machines within Nova, or -containers within Swift. A tenant can have zero or more users, Users can be assocaited with more than one tenant, and each tenant - user pairing can have a role associated with it. +containers within Swift. A tenant can have zero or more users, Users can +be associated with more than one tenant, and each tenant - user pairing can +have a role associated with it. -``tenant create`` +``tenant-create`` ^^^^^^^^^^^^^^^^^ keyword arguments -* tenant_name +* name * description (optional, defaults to None) * enabled (optional, defaults to True) example:: - keystone-manage --id-only tenant create tenant_name=admin + keystone tenant-create --name=demo -creates a tenant named "admin". +creates a tenant named "demo". -``tenant delete`` +``tenant-delete`` ^^^^^^^^^^^^^^^^^ -keyword arguments - -* tenant - -example:: - - keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 - -``tenant update`` -^^^^^^^^^^^^^^^^^ - -keyword arguments - -* tenant_id -* tenant_name (optional, defaults to None) -* description (optional, defaults to None) -* enabled (optional, defaults to True) - -example:: - - keystone-manage tenant update \ - tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 \ - description="those other guys" \ - name=tog - -``tenant get`` -^^^^^^^^^^^^^^ - -keyword arguments +arguments * tenant_id example:: - keystone-manage tenant get \ - tenant_id=523df7c89ce34640996d3d804cbc56f4 + keystone tenant-delete f2b7b39c860840dfa47d9ee4adffa0b3 + +``tenant-enable`` +^^^^^^^^^^^^^^^^^ + +arguments + +* tenant_id + +example:: + + keystone tenant-enable f2b7b39c860840dfa47d9ee4adffa0b3 + +``tenant-disable`` +^^^^^^^^^^^^^^^^^ + +arguments + +* tenant_id + +example:: + + keystone tenant-disable f2b7b39c860840dfa47d9ee4adffa0b3 Users ----- -``user create`` +``user-create`` ^^^^^^^^^^^^^^^ keyword arguments * name -* password +* pass * email -* tenant_id (optional, defaults to None) +* default_tenant (optional, defaults to None) * enabled (optional, defaults to True) example:: - keystone-manage user --id-only create \ - name=admin \ - password=secrete \ - email=admin@example.com + keystone user-create + --name=admin \ + --pass=secrete \ + --email=admin@example.com -.. warning:: - Until https://bugs.launchpad.net/keystone/+bug/927873 is resolved, the - keystone-manage cli doesn't allow the setting enabled to be False, making - this command partially broken at the moment. - -``user delete`` +``user-delete`` ^^^^^^^^^^^^^^^ keyword arguments @@ -226,219 +248,193 @@ keyword arguments example:: - keystone-manage user delete user=f2b7b39c860840dfa47d9ee4adffa0b3 + keystone user-delete f2b7b39c860840dfa47d9ee4adffa0b3 -``user list`` +``user-list`` ^^^^^^^^^^^^^ list users in the system, optionally by a specific tenant (identified by tenant_id) -keyword arguments +arguments * tenant_id (optional, defaults to None) example:: - keystone-manage user list + keystone user-list -``user update_email`` +``user-update-email`` ^^^^^^^^^^^^^^^^^^^^^ -keyword arguments - -* user +arguments +* user_id * email + example:: - keystone-manage user update_email user=03c84b51574841ba9a0d8db7882ac645 email="someone@somewhere.com" + keystone user-update-email 03c84b51574841ba9a0d8db7882ac645 "someone@somewhere.com" - -``user update_enabled`` +``user-enable`` ^^^^^^^^^^^^^^^^^^^^^^^ -keyword arguments +arguments -* user -* enabled (True or False) +* user_id example:: - keystone-manage user update_enabled user=03c84b51574841ba9a0d8db7882ac645 enabled=False + keystone user-enable 03c84b51574841ba9a0d8db7882ac645 -.. warning:: - Until https://bugs.launchpad.net/keystone/+bug/927873 is resolved, the - keystone-manage cli doesn't allow the setting enabled to False, making - this command broken at the moment. +``user-disable`` +^^^^^^^^^^^^^^^^^^^^^^^ + +arguments + +* user_id + +example:: + + keystone user-disable 03c84b51574841ba9a0d8db7882ac645 -``user update_password`` +``user-update-password`` ^^^^^^^^^^^^^^^^^^^^^^^^ -keyword arguments +arguments -* user +* user_id * password example:: - keystone-manage user update_password user=03c84b51574841ba9a0d8db7882ac645 password=foo - -``user update_tenant`` -^^^^^^^^^^^^^^^^^^^^^^ - -keyword arguments - -* user -* tenant - -example:: - - keystone-manage user update_tenant user=03c84b51574841ba9a0d8db7882ac645 tenant=b7b8be32c4be4208949f0373c5909e3b - -``user get`` -^^^^^^^^^^^^ - -keyword arguments - -* user - -example:: - - keystone-manage ususer get user=03c84b51574841ba9a0d8db7882ac645 - + keystone user-update-password 03c84b51574841ba9a0d8db7882ac645 foo Roles ----- -``role create`` +``role-create`` ^^^^^^^^^^^^^^^ -keyword arguments +arguments * name exmaple:: - keystone-manage role --id-only create name=Admin + keystone role-create --name=demo -``role delete`` +``role-delete`` ^^^^^^^^^^^^^^^ -keyword arguments +arguments -* role +* role_id exmaple:: - keystone-manage role delete role=19d1d3344873464d819c45f521ff9890 + keystone role-delete 19d1d3344873464d819c45f521ff9890 -``role list`` +``role-list`` ^^^^^^^^^^^^^^^ exmaple:: - keystone-manage role list + keystone role-list -``role get`` +``role-get`` ^^^^^^^^^^^^ -keysword arguments +arguments -* role +* role_id exmaple:: - keystone-manage role get role=19d1d3344873464d819c45f521ff9890 + keystone role-get role=19d1d3344873464d819c45f521ff9890 -``role add_user_role`` +``add-user-role`` ^^^^^^^^^^^^^^^^^^^^^^ -keyword arguments +arguments -* role -* user -* tenant +* role_id +* user_id +* tenant_id example:: - keystone-manage role add_user_role \ - role=3a751f78ef4c412b827540b829e2d7dd \ - user=03c84b51574841ba9a0d8db7882ac645 \ - tenant=20601a7f1d94447daa4dff438cb1c209 + keystone role add-user-role \ + 3a751f78ef4c412b827540b829e2d7dd \ + 03c84b51574841ba9a0d8db7882ac645 \ + 20601a7f1d94447daa4dff438cb1c209 -``role remove_user_role`` +``remove-user-role`` ^^^^^^^^^^^^^^^^^^^^^^^^^ -keyword arguments +arguments -* role -* user -* tenant (optional, defaults to None) +* role_id +* user_id +* tenant_id example:: - keystone-manage role remove_user_to_tenant \ - role=19d1d3344873464d819c45f521ff9890 \ - user=08741d8ed88242ca88d1f61484a0fe3b \ - tenant=20601a7f1d94447daa4dff438cb1c209 - -``role roles_for_user`` -^^^^^^^^^^^^^^^^^^^^^^^ - -keyword arguments - -* user -* tenant (optional, defaults to None) - -example:: - - keystone-manage role roles_for_user user=08741d8ed88242ca88d1f61484a0fe3b + keystone remove-user-role \ + 19d1d3344873464d819c45f521ff9890 \ + 08741d8ed88242ca88d1f61484a0fe3b \ + 20601a7f1d94447daa4dff438cb1c209 Services -------- -``service create`` +``service-create`` ^^^^^^^^^^^^^^^^^^ keyword arguments * name -* service_type +* type * description example:: - keystone-manage service create \ - name=nova \ - service_type=compute \ - description="Nova Compute Service" + keystone service create \ + --name=nova \ + --type=compute \ + --description="Nova Compute Service" -``service list`` +``service-list`` ^^^^^^^^^^^^^^^^ -keyword arguments +arguments + +* service_id example:: - keystone-manage service list + keystone service-list -``service get`` +``service-get`` ^^^^^^^^^^^^^^^ -keyword arguments +arguments + +* service_id example:: - keystone-manage service get id=08741d8ed88242ca88d1f61484a0fe3b + keystone service-get 08741d8ed88242ca88d1f61484a0fe3b -``service delete`` +``service-delete`` ^^^^^^^^^^^^^^^^^^ -keyword arguments +arguments + +* service_id example:: - keystone-manage service delete id=08741d8ed88242ca88d1f61484a0fe3b + keystone service-delete 08741d8ed88242ca88d1f61484a0fe3b diff --git a/docs/source/configuringservices.rst b/docs/source/configuringservices.rst index 3777ce5ef8..615187eae8 100644 --- a/docs/source/configuringservices.rst +++ b/docs/source/configuringservices.rst @@ -80,21 +80,21 @@ for the OpenStack Dashboard to properly function. Here's how we define the services:: - keystone-manage service create name=nova \ - service_type=compute \ - description="Nova Compute Service" - keystone-manage service create name=ec2 \ - service_type=ec2 \ - description="EC2 Compatibility Layer" - keystone-manage service create name=glance \ - service_type=image \ - description="Glance Image Service" - keystone-manage service create name=keystone \ - service_type=identity \ - description="Keystone Identity Service" - keystone-manage service create name=swift \ - service_type=object-store \ - description="Swift Service" + keystone service-create --name=nova \ + --type=compute \ + --description="Nova Compute Service" + keystone service-create --name=ec2 \ + --type=ec2 \ + --description="EC2 Compatibility Layer" + keystone service-create --name=glance \ + --type=image \ + --description="Glance Image Service" + keystone service-create --name=keystone \ + --type=identity \ + --description="Keystone Identity Service" + keystone service-create --name=swift \ + --type=object-store \ + --description="Swift Service" The endpoints for these services are defined in a template, an example of which is in the project as the file ``etc/default_catalog.templates``. diff --git a/docs/source/man/keystone-manage.rst b/docs/source/man/keystone-manage.rst index da5fc94152..91f2b9e753 100644 --- a/docs/source/man/keystone-manage.rst +++ b/docs/source/man/keystone-manage.rst @@ -22,162 +22,29 @@ DESCRIPTION =========== keystone-manage is the command line tool that interacts with the keystone -service to initialize and update data within Keystone. Keystone *must* be -opertional for the keystone-manage commands to function correctly. +service to initialize and update data within Keystone. Generally, +keystone-manage is only used for operations that can not be accomplished +with through the keystone REST api, such data import/export and schema +migrations. + USAGE ===== - ``keystone-manage [options] type action [additional args]`` + ``keystone-manage [options] action [additional args]`` General keystone-manage options: -------------------------------- -* ``--id-only`` : causes ``keystone-manage`` to return only the UUID result -from the API call. -* ``--endpoint`` : allows you to specify the keystone endpoint to communicate with. The default endpoint is http://localhost:35357/v2.0' -* ``--auth-token`` : provides the authorization token - -``keystone-manage`` is set up to expect commands in the general form of ``keystone-manage`` ``command`` ``subcommand``, with keyword arguments to provide additional information to the command. For example, the command -``tenant`` has the subcommand ``create``, which takes the required keyword ``tenant_name``:: - - keystone-manage tenant create tenant_name=example_tenant +* ``--help`` : display verbose help output. Invoking keystone-manage by itself will give you some usage information. Available keystone-manage commands: db_sync: Sync the database. - ec2: no docs - role: Role CRUD functions. - service: Service CRUD functions. - tenant: Tenant CRUD functions. - token: Token CRUD functions. - user: User CRUD functions. - -Tenants -------- - -Tenants are the high level grouping within Keystone that represent groups of -users. A tenant is the grouping that owns virtual machines within Nova, or -containers within Swift. A tenant can have zero or more users, Users can be assocaited with more than one tenant, and each tenant - user pairing can have a role associated with it. - -* tenant create - - keyword arguments - * tenant_name - * id (optional) - -example:: - keystone-manage --id-only tenant create tenant_name=admin - -creates a tenant named "admin". - -* tenant delete - - keyword arguments - * tenant_id - -example:: - keystone-manage tenant delete tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 - -* tenant update - - keyword arguments - * description - * name - * tenant_id - -example:: - keystone-manage tenant update \ - tenant_id=f2b7b39c860840dfa47d9ee4adffa0b3 \ - description="those other guys" \ - name=tog - -Users ------ - -* user create - - keyword arguments - * name - * password - * email - -example:: - keystone-manage user --ks-id-only create \ - name=admin \ - password=secrete \ - email=admin@example.com - -* user delete - - keyword arguments - -* user list - - keyword arguments - -* user update_email - - keyword arguments - -* user update_enabled - - keyword arguments - -* user update_password - - keyword arguments - -* user update_tenant - - keyword arguments - -Roles ------ - -* role create - - keyword arguments - * name - -exmaple:: - keystone-manage role --ks-id-only create name=Admin - -* role add_user_to_tenant - - keyword arguments - * role_id - * user_id - * tenant_id - -example:: - - keystone-manage role add_user_to_tenant \ - role_id=19d1d3344873464d819c45f521ff9890 \ - user_id=08741d8ed88242ca88d1f61484a0fe3b \ - tenant_id=20601a7f1d94447daa4dff438cb1c209 - -* role remove_user_from_tenant - -* role get_user_role_refs - -Services --------- - -* service create - - keyword arguments - * name - * service_type - * description - -example:: - keystone-manage service create \ - name=nova \ - service_type=compute \ - description="Nova Compute Service" + import_legacy: Import a legacy (pre-essex) version of the db. + export_legacy_catalog: Export service catalog from a legacy (pre-essex) db. OPTIONS @@ -212,11 +79,6 @@ Options: syslog (defaults to LOG_USER) --use-syslog Use syslog for logging. --nouse-syslog Use syslog for logging. - --endpoint=ENDPOINT - --auth-token=AUTH_TOKEN - authorization token - --id-only - --noid-only FILES ===== From 71436dbf188b3ff1c576fcd54b992530aac98b6c Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 8 Feb 2012 16:08:08 -0800 Subject: [PATCH 333/334] Add token expiration * Config option token.expiration defines amount of time tokens should be valid * Fixes bug 928545 Change-Id: I3dff7a1ebf03bb44fc6e5247f976baea0581de08 --- etc/keystone.conf | 3 ++ keystone/common/sql/core.py | 1 + .../versions/001_add_initial_tables.py | 1 + keystone/common/utils.py | 24 ++++++++++++++ keystone/contrib/ec2/core.py | 3 +- keystone/identity/backends/sql.py | 4 ++- keystone/service.py | 12 ++++--- keystone/token/backends/kvs.py | 18 +++++++--- keystone/token/backends/memcache.py | 13 ++++++-- keystone/token/backends/sql.py | 33 +++++++++++++------ keystone/token/core.py | 12 +++++++ tests/backend_sql.conf | 3 ++ tests/test_backend.py | 22 +++++++++++++ tests/test_backend_memcache.py | 14 +++++--- tests/test_keystoneclient_sql.py | 1 - tests/test_utils.py | 9 +++++ 16 files changed, 142 insertions(+), 31 deletions(-) diff --git a/etc/keystone.conf b/etc/keystone.conf index fde1300487..dd78b4a957 100644 --- a/etc/keystone.conf +++ b/etc/keystone.conf @@ -32,6 +32,9 @@ template_file = ./etc/default_catalog.templates [token] driver = keystone.token.backends.kvs.Token +# Amount of time a token should remain valid (in seconds) +expiration = 86400 + [policy] driver = keystone.policy.backends.simple.SimpleMatch diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index 2bd24f4835..cb62186598 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -26,6 +26,7 @@ ModelBase = declarative.declarative_base() Column = sql.Column String = sql.String ForeignKey = sql.ForeignKey +DateTime = sql.DateTime # Special Fields diff --git a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py index 5875817e5b..ae54b476dc 100644 --- a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py +++ b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py @@ -5,6 +5,7 @@ from keystone.common import sql # these are to make sure all the models we care about are defined import keystone.identity.backends.sql +import keystone.token.backends.sql import keystone.contrib.ec2.backends.sql diff --git a/keystone/common/utils.py b/keystone/common/utils.py index b5269bed2f..96e595bb0a 100644 --- a/keystone/common/utils.py +++ b/keystone/common/utils.py @@ -23,6 +23,7 @@ import hmac import json import subprocess import sys +import time import urllib import passlib.hash @@ -35,6 +36,9 @@ CONF = config.CONF config.register_int('crypt_strength', default=40000) +ISO_TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + + def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') @@ -201,3 +205,23 @@ def check_output(*popenargs, **kwargs): def git(*args): return check_output(['git'] + list(args)) + + +def isotime(dt_obj): + """Format datetime object as ISO compliant string. + + :param dt_obj: datetime.datetime object + :returns: string representation of datetime object + + """ + return dt_obj.strftime(ISO_TIME_FORMAT) + + +def unixtime(dt_obj): + """Format datetime object as unix timestamp + + :param dt_obj: datetime.datetime object + :returns: float + + """ + return time.mktime(dt_obj.utctimetuple()) diff --git a/keystone/contrib/ec2/core.py b/keystone/contrib/ec2/core.py index c8ad4425b6..08a286cc60 100644 --- a/keystone/contrib/ec2/core.py +++ b/keystone/contrib/ec2/core.py @@ -163,8 +163,7 @@ class Ec2Controller(wsgi.Application): metadata=metadata_ref) token_ref = self.token_api.create_token( - context, token_id, dict(expires='', - id=token_id, + context, token_id, dict(id=token_id, user=user_ref, tenant=tenant_ref, metadata=metadata_ref)) diff --git a/keystone/identity/backends/sql.py b/keystone/identity/backends/sql.py index 8a3db42248..4918a94241 100644 --- a/keystone/identity/backends/sql.py +++ b/keystone/identity/backends/sql.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import copy + from keystone import identity from keystone.common import sql from keystone.common import utils @@ -64,7 +66,7 @@ class Tenant(sql.ModelBase, sql.DictBase): return cls(**tenant_dict) def to_dict(self): - extra_copy = self.extra.copy() + extra_copy = copy.deepcopy(self.extra) extra_copy['id'] = self.id extra_copy['name'] = self.name return extra_copy diff --git a/keystone/service.py b/keystone/service.py index 5be853c1bc..2c361e400b 100644 --- a/keystone/service.py +++ b/keystone/service.py @@ -12,6 +12,7 @@ from keystone import identity from keystone import policy from keystone import token from keystone.common import logging +from keystone.common import utils from keystone.common import wsgi @@ -226,8 +227,7 @@ class TokenController(wsgi.Application): raise webob.exc.HTTPForbidden(e.message) token_ref = self.token_api.create_token( - context, token_id, dict(expires='', - id=token_id, + context, token_id, dict(id=token_id, user=user_ref, tenant=tenant_ref, metadata=metadata_ref)) @@ -283,8 +283,7 @@ class TokenController(wsgi.Application): catalog_ref = {} token_ref = self.token_api.create_token( - context, token_id, dict(expires='', - id=token_id, + context, token_id, dict(id=token_id, user=user_ref, tenant=tenant_ref, metadata=metadata_ref)) @@ -351,8 +350,11 @@ class TokenController(wsgi.Application): def _format_token(self, token_ref, roles_ref): user_ref = token_ref['user'] metadata_ref = token_ref['metadata'] + expires = token_ref['expires'] + if expires is not None: + expires = utils.isotime(expires) o = {'access': {'token': {'id': token_ref['id'], - 'expires': token_ref['expires'] + 'expires': expires, }, 'user': {'id': user_ref['id'], 'name': user_ref['name'], diff --git a/keystone/token/backends/kvs.py b/keystone/token/backends/kvs.py index 84604ac5e3..990e107bee 100644 --- a/keystone/token/backends/kvs.py +++ b/keystone/token/backends/kvs.py @@ -1,5 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import copy +import datetime + from keystone.common import kvs from keystone import exception from keystone import token @@ -8,14 +11,19 @@ from keystone import token class Token(kvs.Base, token.Driver): # Public interface def get_token(self, token_id): - try: - return self.db['token-%s' % token_id] - except KeyError: + token = self.db.get('token-%s' % token_id) + if (token and (token['expires'] is None + or token['expires'] > datetime.datetime.now())): + return token + else: raise exception.TokenNotFound(token_id=token_id) def create_token(self, token_id, data): - self.db.set('token-%s' % token_id, data) - return data + data_copy = copy.deepcopy(data) + if 'expires' not in data: + data_copy['expires'] = self._get_default_expire_time() + self.db.set('token-%s' % token_id, data_copy) + return copy.deepcopy(data_copy) def delete_token(self, token_id): try: diff --git a/keystone/token/backends/memcache.py b/keystone/token/backends/memcache.py index 8dfb373197..b9c2cbe1dd 100644 --- a/keystone/token/backends/memcache.py +++ b/keystone/token/backends/memcache.py @@ -1,12 +1,14 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 from __future__ import absolute_import +import copy import memcache from keystone import config from keystone import exception from keystone import token +from keystone.common import utils CONF = config.CONF @@ -38,9 +40,16 @@ class Token(token.Driver): return token def create_token(self, token_id, data): + data_copy = copy.deepcopy(data) ptk = self._prefix_token_id(token_id) - self.client.set(ptk, data) - return data + if 'expires' not in data_copy: + data_copy['expires'] = self._get_default_expire_time() + kwargs = {} + if data_copy['expires'] is not None: + expires_ts = utils.unixtime(data_copy['expires']) + kwargs['time'] = expires_ts + self.client.set(ptk, data_copy, **kwargs) + return copy.deepcopy(data_copy) def delete_token(self, token_id): # Test for existence diff --git a/keystone/token/backends/sql.py b/keystone/token/backends/sql.py index 090a3600dd..b57f32a876 100644 --- a/keystone/token/backends/sql.py +++ b/keystone/token/backends/sql.py @@ -1,5 +1,8 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import copy +import datetime + from keystone.common import sql from keystone import exception from keystone import token @@ -8,21 +11,24 @@ from keystone import token class TokenModel(sql.ModelBase, sql.DictBase): __tablename__ = 'token' id = sql.Column(sql.String(64), primary_key=True) + expires = sql.Column(sql.DateTime(), default=None) extra = sql.Column(sql.JsonBlob()) @classmethod def from_dict(cls, token_dict): # shove any non-indexed properties into extra + extra = copy.deepcopy(token_dict) data = {} - token_dict_copy = token_dict.copy() - data['id'] = token_dict_copy.pop('id') - data['extra'] = token_dict_copy + for k in ('id', 'expires'): + data[k] = extra.pop(k, None) + data['extra'] = extra return cls(**data) def to_dict(self): - extra_copy = self.extra.copy() - extra_copy['id'] = self.id - return extra_copy + out = copy.deepcopy(self.extra) + out['id'] = self.id + out['expires'] = self.expires + return out class Token(sql.Base, token.Driver): @@ -30,15 +36,22 @@ class Token(sql.Base, token.Driver): def get_token(self, token_id): session = self.get_session() token_ref = session.query(TokenModel).filter_by(id=token_id).first() - if not token_ref: + now = datetime.datetime.now() + if token_ref and (not token_ref.expires or now < token_ref.expires): + return token_ref.to_dict() + else: raise exception.TokenNotFound(token_id=token_id) - return token_ref.to_dict() def create_token(self, token_id, data): - data['id'] = token_id + data_copy = copy.deepcopy(data) + if 'expires' not in data_copy: + data_copy['expires'] = self._get_default_expire_time() + + token_ref = TokenModel.from_dict(data_copy) + token_ref.id = token_id + session = self.get_session() with session.begin(): - token_ref = TokenModel.from_dict(data) session.add(token_ref) session.flush() return token_ref.to_dict() diff --git a/keystone/token/core.py b/keystone/token/core.py index 7183c1793e..8c816efe2b 100644 --- a/keystone/token/core.py +++ b/keystone/token/core.py @@ -2,11 +2,14 @@ """Main entry point into the Token service.""" +import datetime + from keystone import config from keystone.common import manager CONF = config.CONF +config.register_int('expiration', group='token', default=86400) class Manager(manager.Manager): @@ -68,3 +71,12 @@ class Driver(object): """ raise NotImplementedError() + + def _get_default_expire_time(self): + """Determine when a token should expire based on the config. + + :returns: datetime.datetime object + + """ + expire_delta = datetime.timedelta(seconds=CONF.token.expiration) + return datetime.datetime.now() + expire_delta diff --git a/tests/backend_sql.conf b/tests/backend_sql.conf index 0fec0488a4..cb246194b1 100644 --- a/tests/backend_sql.conf +++ b/tests/backend_sql.conf @@ -8,5 +8,8 @@ pool_timeout = 200 [identity] driver = keystone.identity.backends.sql.Identity +[token] +driver = keystone.token.backends.sql.Token + [ec2] driver = keystone.contrib.ec2.backends.sql.Ec2 diff --git a/tests/test_backend.py b/tests/test_backend.py index 9dc949da57..bbc651ef9f 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,5 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import datetime import uuid from keystone import exception @@ -211,9 +212,13 @@ class TokenTests(object): token_id = uuid.uuid4().hex data = {'id': token_id, 'a': 'b'} data_ref = self.token_api.create_token(token_id, data) + expires = data_ref.pop('expires') + self.assertTrue(isinstance(expires, datetime.datetime)) self.assertDictEquals(data_ref, data) new_data_ref = self.token_api.get_token(token_id) + expires = new_data_ref.pop('expires') + self.assertTrue(isinstance(expires, datetime.datetime)) self.assertEquals(new_data_ref, data) self.token_api.delete_token(token_id) @@ -221,3 +226,20 @@ class TokenTests(object): self.token_api.delete_token, token_id) self.assertRaises(exception.TokenNotFound, self.token_api.get_token, token_id) + + def test_expired_token(self): + token_id = uuid.uuid4().hex + expire_time = datetime.datetime.now() - datetime.timedelta(minutes=1) + data = {'id': token_id, 'a': 'b', 'expires': expire_time} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + self.assertRaises(exception.TokenNotFound, + self.token_api.get_token, token_id) + + def test_null_expires_token(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, 'a': 'b', 'expires': None} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + new_data_ref = self.token_api.get_token(token_id) + self.assertEqual(data_ref, new_data_ref) diff --git a/tests/test_backend_memcache.py b/tests/test_backend_memcache.py index b320b1f91a..05ef2107c8 100644 --- a/tests/test_backend_memcache.py +++ b/tests/test_backend_memcache.py @@ -1,5 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +import datetime +import time import uuid import memcache @@ -25,15 +27,17 @@ class MemcacheClient(object): def get(self, key): """Retrieves the value for a key or None.""" self.check_key(key) - try: - return self.cache[key] - except KeyError: + obj = self.cache.get(key) + now = time.mktime(datetime.datetime.now().timetuple()) + if obj and (obj[1] == 0 or obj[1] > now): + return obj[0] + else: raise exception.TokenNotFound(token_id=key) - def set(self, key, value): + def set(self, key, value, time=0): """Sets the value for a key.""" self.check_key(key) - self.cache[key] = value + self.cache[key] = (value, time) return True def delete(self, key): diff --git a/tests/test_keystoneclient_sql.py b/tests/test_keystoneclient_sql.py index e365f83c5e..0d8ef52dfa 100644 --- a/tests/test_keystoneclient_sql.py +++ b/tests/test_keystoneclient_sql.py @@ -2,7 +2,6 @@ from keystone import config from keystone import test from keystone.common.sql import util as sql_util -from keystone.common.sql import migration import test_keystoneclient diff --git a/tests/test_utils.py b/tests/test_utils.py index 81cec7c918..7409e30c65 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime + from keystone import test from keystone.common import utils @@ -38,3 +40,10 @@ class UtilsTestCase(test.TestCase): hashed = utils.hash_password(password) self.assertTrue(utils.check_password(password, hashed)) self.assertFalse(utils.check_password(wrong, hashed)) + + def test_isotime(self): + dt = datetime.datetime(year=1987, month=10, day=13, + hour=1, minute=2, second=3) + output = utils.isotime(dt) + expected = '1987-10-13T01:02:03Z' + self.assertEqual(output, expected) From 90068b0143af788869116d08533d5ebc99874a17 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Tue, 14 Feb 2012 14:58:55 -0800 Subject: [PATCH 334/334] Add docs on keystone_old -> ksl migration * Fixes bug 928046 Change-Id: I4af516dbc9577c08a77850e75e45d98040e4fb27 --- docs/source/configuration.rst | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 7b975b1082..22f3748b1f 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -91,6 +91,54 @@ Invoking this command starts up two wsgi.Server instances, configured by the ``admin`` (the administration API) and the other is ``main`` (the primary/public API interface). Both of these run in a single process. +Migrating from legacy versions of keystone +========================================== +Migration support is provided for the following legacy keystone versions: + +* diablo-5 +* stable/diablo +* essex-2 +* essex-3 + +To migrate from legacy versions of keystone, use the following steps: + +Step 1: Configure keystone.conf +------------------------------- +It is important that the database that you specify be different from the one +containing your existing install. + +Step 2: db_sync your new, empty database +---------------------------------------- +Run the following command to configure the most recent schema in your new +keystone installation:: + + keystone-manage db_sync + +Step 3: Import your legacy data +------------------------------- +Use the following command to import your old data:: + + keystone-manage import_legacy [db_url, e.g. 'mysql://root@foobar/keystone'] + +Specify db_url as the connection string that was present in your old +keystone.conf file. + +Step 3: Import your legacy service catalog +------------------------------------------ +While the older keystone stored the service catalog in the database, +the updated version configures the service catalog using a template file. +An example service catalog template file may be found in +etc/default_catalog.templates. + +To import your legacy catalog, run this command:: + + keystone-manage export_legacy_catalog \ + [db_url e.g. 'mysql://root@foobar/keystone'] > \ + [path_to_templates e.g. 'etc/default_catalog.templates'] + +After executing this command, you will need to restart the keystone service to +see your changes. + Initializing Keystone =====================