From da29b6044626e7fc854c772f038e54ac297467da Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 28 Jul 2010 22:41:49 -0700 Subject: [PATCH 01/11] Fix deprecation warning in AuthManager. __new__ isn't allowed to take args. --- nova/auth/manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2da53a73..2360c1a5 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -24,7 +24,6 @@ import logging import os import shutil import string -import sys import tempfile import uuid import zipfile @@ -322,11 +321,10 @@ class AuthManager(object): need to be more accessible, such as vpn ips and ports. """ _instance=None - def __new__(cls, *args, **kwargs): + def __new__(cls): """Returns the AuthManager singleton""" if not cls._instance: - cls._instance = super(AuthManager, cls).__new__( - cls, *args, **kwargs) + cls._instance = super(AuthManager, cls).__new__(cls) return cls._instance def __init__(self, driver=None, *args, **kwargs): From 9a393bfd043c763fbfaaccf304e18efb5015fd67 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 28 Jul 2010 23:19:07 -0700 Subject: [PATCH 02/11] oops retry and add extra exception check --- nova/auth/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2360c1a5..b690176b 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -321,7 +321,7 @@ class AuthManager(object): need to be more accessible, such as vpn ips and ports. """ _instance=None - def __new__(cls): + def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" if not cls._instance: cls._instance = super(AuthManager, cls).__new__(cls) From d7be65b0b0c55411e335d15944554ae06faa0d0c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 29 Jul 2010 16:52:08 -0700 Subject: [PATCH 03/11] fix imports in endpoint/images.py boto.s3 no longer imports connection, so we need to explicitly import it. --- nova/endpoint/images.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index 12876da3..32f7cc22 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -21,10 +21,8 @@ Proxy AMI-related calls from the cloud controller, to the running objectstore daemon. """ -import boto -import boto.s3 +import boto.s3.connection import json -import random import urllib from nova import flags From b0986a25d504e36aecf49ce24eab27519666db3f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 29 Jul 2010 17:10:28 -0700 Subject: [PATCH 04/11] boto.utils import doesn't work with new boto, import boto instead --- nova/auth/signer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/auth/signer.py b/nova/auth/signer.py index 7d747157..3b9bc8f2 100644 --- a/nova/auth/signer.py +++ b/nova/auth/signer.py @@ -34,7 +34,7 @@ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. @@ -48,7 +48,7 @@ import hashlib import hmac import logging import urllib -import boto.utils +import boto from nova.exception import Error From 1065ba807cc6aa5e48e5ed0bd48739118440eb70 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 29 Jul 2010 17:18:39 -0700 Subject: [PATCH 05/11] Added project param to admin client zip download --- nova/adminclient.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index fceeac27..25d5e71c 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -292,9 +292,13 @@ class NovaAdminClient(object): 'Operation': operation} return self.apiconn.get_status('ModifyProjectMember', params) - def get_zip(self, username): - """ returns the content of a zip file containing novarc and access credentials. """ - return self.apiconn.get_object('GenerateX509ForUser', {'Name': username}, UserInfo).file + def get_zip(self, user, project): + """ + Returns the content of a zip file containing novarc and access credentials. + """ + params = {'Name': user, 'Project': project} + zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo) + return zip.file def get_hosts(self): return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) From c949ff0b02b83f8a0554b8dcac59188dd50448e4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 11:29:13 -0700 Subject: [PATCH 06/11] Make nodaemon twistd processes log to stdout --- bin/nova-objectstore | 5 ----- nova/twistd.py | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 9385fd29..c0fa815c 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -21,8 +21,6 @@ Twisted daemon for nova objectstore. Supports S3 API. """ -import logging - from nova import flags from nova import utils from nova import twistd @@ -33,9 +31,6 @@ FLAGS = flags.FLAGS def main(): - # FIXME: if this log statement isn't here, no logging - # appears from other files and app won't start daemonized - logging.debug('Started HTTP server on %s' % (FLAGS.s3_port)) app = handler.get_application() print app return app diff --git a/nova/twistd.py b/nova/twistd.py index ecb6e289..c83276da 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -214,6 +214,9 @@ def serve(filename): FLAGS.pidfile = '%s.pid' % name elif FLAGS.pidfile.endswith('twistd.pid'): FLAGS.pidfile = FLAGS.pidfile.replace('twistd.pid', '%s.pid' % name) + # NOTE(vish): if we're running nodaemon, redirect the log to stdout + if FLAGS.nodaemon and not FLAGS.logfile: + FLAGS.logfile = "-" if not FLAGS.logfile: FLAGS.logfile = '%s.log' % name elif FLAGS.logfile.endswith('twistd.log'): From ac0c78dda342b03a01cbe4ba0187ee243811ca5d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 15:19:41 -0700 Subject: [PATCH 07/11] Fixes access key passing in curl statement. --- nova/auth/manager.py | 4 ++++ nova/endpoint/images.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2da53a73..ca9f4fc8 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -419,6 +419,10 @@ class AuthManager(object): raise exception.NotAuthorized('Signature does not match') return (user, project) + def get_access_key(self, user, project): + """Get an access key that includes user and project""" + return "%s:%s" % (User.safe_id(user), Project.safe_id(project)) + def is_superuser(self, user): """Checks for superuser status, allowing user to bypass rbac diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index 32f7cc22..fe7cb5d1 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -27,6 +27,7 @@ import urllib from nova import flags from nova import utils +from nova.auth import manager FLAGS = flags.FLAGS @@ -75,13 +76,16 @@ def deregister(context, image_id): query_args=qs({'image_id': image_id})) def conn(context): - return boto.s3.connection.S3Connection ( - aws_access_key_id=str('%s:%s' % (context.user.access, context.project.name)), - aws_secret_access_key=str(context.user.secret), - is_secure=False, - calling_format=boto.s3.connection.OrdinaryCallingFormat(), - port=FLAGS.s3_port, - host=FLAGS.s3_host) + access = manager.AuthManager().get_access_key(context.user, + context.project) + secret = str(context.user.secret) + calling = boto.s3.connection.OrdinaryCallingFormat() + return boto.s3.connection.S3Connection(aws_access_key_id=access, + aws_secret_access_key=secret, + is_secure=False, + calling_format=calling, + port=FLAGS.s3_port, + host=FLAGS.s3_host) def qs(params): From 6ea0542c123c5b47c50174025caff36a4e7dd71a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 15:36:11 -0700 Subject: [PATCH 08/11] use user.access instead of user.id --- nova/auth/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index ca9f4fc8..bf3a3556 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -421,7 +421,9 @@ class AuthManager(object): def get_access_key(self, user, project): """Get an access key that includes user and project""" - return "%s:%s" % (User.safe_id(user), Project.safe_id(project)) + if not isinstance(user, User): + user = self.get_user(user) + return "%s:%s" % (user.access, Project.safe_id(project)) def is_superuser(self, user): """Checks for superuser status, allowing user to bypass rbac From f3d918ff1e6942aa747d2bb48a1d7e609361ce64 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 16:15:09 -0700 Subject: [PATCH 09/11] another try on fix boto --- nova/auth/signer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/auth/signer.py b/nova/auth/signer.py index 3b9bc8f2..634f22f0 100644 --- a/nova/auth/signer.py +++ b/nova/auth/signer.py @@ -48,7 +48,8 @@ import hashlib import hmac import logging import urllib -import boto +import boto # NOTE(vish): for new boto +import boto.utils # NOTE(vish): for old boto from nova.exception import Error From 9b11290b7d2e07bfe1069efe6e4fa800bbfd6070 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 18:32:12 -0700 Subject: [PATCH 10/11] Fixes nova volumes. The async commands yield properly. Simplified the call to create volume in cloud. Added some notes --- nova/endpoint/cloud.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 67fc0450..0ee278f8 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -295,17 +295,16 @@ class CloudController(object): return v @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it - res = rpc.call(FLAGS.volume_topic, {"method": "create_volume", + result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", "args" : {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) - def _format_result(result): - volume = self._get_volume(context, result['result']) - return {'volumeSet': [self.format_volume(context, volume)]} - res.addCallback(_format_result) - return res + # NOTE(vish): rpc returned value is in the result key in the dictionary + volume = self._get_volume(context, result['result']) + defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py From 523833742c9365e3e0f2c7520f26c27c844b1582 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 19:33:07 -0700 Subject: [PATCH 11/11] Fix Tests --- nova/tests/volume_unittest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index b536ac38..0f4f0e34 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -42,15 +42,14 @@ class VolumeTestCase(test.TrialTestCase): vol_size = '0' user_id = 'fake' project_id = 'fake' - volume_id = self.volume.create_volume(vol_size, user_id, project_id) + volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) # TODO(termie): get_volume returns differently than create_volume self.assertEqual(volume_id, volume_service.get_volume(volume_id)['volume_id']) rv = self.volume.delete_volume(volume_id) - self.assertRaises(exception.Error, - volume_service.get_volume, - volume_id) + self.assertFailure(volume_service.get_volume(volume_id), + exception.Error) def test_too_big_volume(self): vol_size = '1001' @@ -68,13 +67,14 @@ class VolumeTestCase(test.TrialTestCase): total_slots = FLAGS.slots_per_shelf * num_shelves vols = [] for i in xrange(total_slots): - vid = self.volume.create_volume(vol_size, user_id, project_id) + vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) - self.assertRaises(volume_service.NoMoreVolumes, - self.volume.create_volume, - vol_size, user_id, project_id) + self.assertFailure(self.volume.create_volume(vol_size, + user_id, + project_id), + volume_service.NoMoreVolumes) for id in vols: - self.volume.delete_volume(id) + yield self.volume.delete_volume(id) def test_run_attach_detach_volume(self): # Create one volume and one compute to test with @@ -83,7 +83,7 @@ class VolumeTestCase(test.TrialTestCase): user_id = "fake" project_id = 'fake' mountpoint = "/dev/sdf" - volume_id = self.volume.create_volume(vol_size, user_id, project_id) + volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) volume_obj = volume_service.get_volume(volume_id) volume_obj.start_attach(instance_id, mountpoint)