From ac0edc5ee11b86e1035c8ac567f68892d0122efb Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Tue, 10 Aug 2010 09:55:00 -0400 Subject: [PATCH 001/113] Pylint fixes for /nova/tests/api_unittest.py --- nova/tests/api_unittest.py | 83 ++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 9d072866..462d1b29 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +"""Unit tests for the API endpoint""" + import boto from boto.ec2 import regioninfo import httplib @@ -38,7 +40,15 @@ FLAGS = flags.FLAGS # circuit boto calls and feed them into our tornado handlers, # it's pretty damn circuitous so apologies if you have to fix # a bug in it -def boto_to_tornado(method, path, headers, data, host, connection=None): +# NOTE(jaypipes) The pylint disables here are for R0913 (too many args) which +# isn't controllable since boto's HTTPRequest needs that many +# args, and for the version-differentiated import of tornado's +# httputil. +# NOTE(jaypipes): The disable-msg=E1101 and E1103 below is because pylint is +# unable to introspect the deferred's return value properly + +def boto_to_tornado(method, path, headers, data, # pylint: disable-msg=R0913 + host, connection=None): """ translate boto requests into tornado requests connection should be a FakeTornadoHttpConnection instance @@ -46,7 +56,7 @@ def boto_to_tornado(method, path, headers, data, host, connection=None): try: headers = httpserver.HTTPHeaders() except AttributeError: - from tornado import httputil + from tornado import httputil # pylint: disable-msg=E0611 headers = httputil.HTTPHeaders() for k, v in headers.iteritems(): headers[k] = v @@ -61,57 +71,64 @@ def boto_to_tornado(method, path, headers, data, host, connection=None): return req -def raw_to_httpresponse(s): - """ translate a raw tornado http response into an httplib.HTTPResponse """ - sock = FakeHttplibSocket(s) +def raw_to_httpresponse(response_string): + """translate a raw tornado http response into an httplib.HTTPResponse""" + sock = FakeHttplibSocket(response_string) resp = httplib.HTTPResponse(sock) resp.begin() return resp class FakeHttplibSocket(object): - """ a fake socket implementation for httplib.HTTPResponse, trivial """ - def __init__(self, s): - self.fp = StringIO.StringIO(s) + """a fake socket implementation for httplib.HTTPResponse, trivial""" + def __init__(self, response_string): + self._buffer = StringIO.StringIO(response_string) - def makefile(self, mode, other): - return self.fp + def makefile(self, _mode, _other): + """Returns the socket's internal buffer""" + return self._buffer class FakeTornadoStream(object): - """ a fake stream to satisfy tornado's assumptions, trivial """ - def set_close_callback(self, f): + """a fake stream to satisfy tornado's assumptions, trivial""" + def set_close_callback(self, _func): + """Dummy callback for stream""" pass class FakeTornadoConnection(object): - """ a fake connection object for tornado to pass to its handlers + """A fake connection object for tornado to pass to its handlers web requests are expected to write to this as they get data and call finish when they are done with the request, we buffer the writes and kick off a callback when it is done so that we can feed the result back into boto. """ - def __init__(self, d): - self.d = d + def __init__(self, deferred): + self._deferred = deferred self._buffer = StringIO.StringIO() def write(self, chunk): + """Writes a chunk of data to the internal buffer""" self._buffer.write(chunk) def finish(self): - s = self._buffer.getvalue() - self.d.callback(s) + """Finalizes the connection and returns the buffered data via the + deferred callback. + """ + data = self._buffer.getvalue() + self._deferred.callback(data) xheaders = None @property - def stream(self): + def stream(self): # pylint: disable-msg=R0201 + """Required property for interfacing with tornado""" return FakeTornadoStream() class FakeHttplibConnection(object): - """ a fake httplib.HTTPConnection for boto to use + """A fake httplib.HTTPConnection for boto to use requests made via this connection actually get translated and routed into our tornado app, we then wait for the response and turn it back into @@ -123,7 +140,9 @@ class FakeHttplibConnection(object): self.deferred = defer.Deferred() def request(self, method, path, data, headers): - req = boto_to_tornado + """Creates a connection to a fake tornado and sets + up a deferred request with the supplied data and + headers""" conn = FakeTornadoConnection(self.deferred) request = boto_to_tornado(connection=conn, method=method, @@ -131,12 +150,16 @@ class FakeHttplibConnection(object): headers=headers, data=data, host=self.host) - handler = self.app(request) + self.app(request) self.deferred.addCallback(raw_to_httpresponse) def getresponse(self): + """A bit of deferred magic for catching the response + from the previously deferred request""" @defer.inlineCallbacks def _waiter(): + """Callback that simply yields the deferred's + return value.""" result = yield self.deferred defer.returnValue(result) d = _waiter() @@ -144,14 +167,16 @@ class FakeHttplibConnection(object): # this deferred has already been called by the time # we get here, we are going to cheat and return # the result of the callback - return d.result + return d.result # pylint: disable-msg=E1101 def close(self): + """Required for compatibility with boto/tornado""" pass class ApiEc2TestCase(test.BaseTestCase): - def setUp(self): + """Unit test for the cloud controller on an EC2 API""" + def setUp(self): # pylint: disable-msg=C0103,C0111 super(ApiEc2TestCase, self).setUp() self.manager = manager.AuthManager() @@ -171,12 +196,16 @@ class ApiEc2TestCase(test.BaseTestCase): self.mox.StubOutWithMock(self.ec2, 'new_http_connection') def expect_http(self, host=None, is_secure=False): + """Returns a new EC2 connection""" http = FakeHttplibConnection( self.app, '%s:%d' % (self.host, FLAGS.cc_port), False) + # pylint: disable-msg=E1103 self.ec2.new_http_connection(host, is_secure).AndReturn(http) return http def test_describe_instances(self): + """Test that, after creating a user and a project, the describe + instances call to the API works properly""" self.expect_http() self.mox.ReplayAll() user = self.manager.create_user('fake', 'fake', 'fake') @@ -187,14 +216,18 @@ class ApiEc2TestCase(test.BaseTestCase): def test_get_all_key_pairs(self): + """Test that, after creating a user and project and generating + a key pair, that the API call to list key pairs works properly""" self.expect_http() self.mox.ReplayAll() - keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) + keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + for x in range(random.randint(4, 8))) user = self.manager.create_user('fake', 'fake', 'fake') project = self.manager.create_project('fake', 'fake', 'fake') self.manager.generate_key_pair(user.id, keyname) rv = self.ec2.get_all_key_pairs() - self.assertTrue(filter(lambda k: k.name == keyname, rv)) + results = [k for k in rv if k.name == keyname] + self.assertEquals(len(results), 1) self.manager.delete_project(project) self.manager.delete_user(user) From 653f5470c3f56a7795d5eecc1d7f0fedfef2dbd1 Mon Sep 17 00:00:00 2001 From: Sleepsonthefloor Date: Sat, 14 Aug 2010 02:13:12 -0700 Subject: [PATCH 002/113] initial commit for orm based models --- nova/auth.py | 741 +++++++++++++++++++++++++++++++++++++++++++++++++ nova/models.py | 198 +++++++++++++ 2 files changed, 939 insertions(+) create mode 100644 nova/auth.py create mode 100644 nova/models.py diff --git a/nova/auth.py b/nova/auth.py new file mode 100644 index 00000000..199a887e --- /dev/null +++ b/nova/auth.py @@ -0,0 +1,741 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Nova authentication management +""" + +import logging +import os +import shutil +import string +import tempfile +import uuid +import zipfile + +from nova import crypto +from nova import exception +from nova import flags +from nova import utils +from nova.auth import signer +from nova.network import vpn +from nova.models import User + +#unused imports +#from nova import datastore +#from nova.auth import ldapdriver # for flags +#from nova import objectstore # for flags + +FLAGS = flags.FLAGS + +# NOTE(vish): a user with one of these roles will be a superuser and +# have access to all api commands +flags.DEFINE_list('superuser_roles', ['cloudadmin'], + 'Roles that ignore rbac checking completely') + +# NOTE(vish): a user with one of these roles will have it for every +# project, even if he or she is not a member of the project +flags.DEFINE_list('global_roles', ['cloudadmin', 'itsec'], + 'Roles that apply to all projects') + + +flags.DEFINE_string('credentials_template', + utils.abspath('auth/novarc.template'), + 'Template for creating users rc file') +flags.DEFINE_string('vpn_client_template', + utils.abspath('cloudpipe/client.ovpn.template'), + 'Template for creating users vpn file') +flags.DEFINE_string('credential_vpn_file', 'nova-vpn.conf', + 'Filename of certificate in credentials zip') +flags.DEFINE_string('credential_key_file', 'pk.pem', + 'Filename of private key in credentials zip') +flags.DEFINE_string('credential_cert_file', 'cert.pem', + 'Filename of certificate in credentials zip') +flags.DEFINE_string('credential_rc_file', 'novarc', + 'Filename of rc in credentials zip') + +flags.DEFINE_string('credential_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=%s-%s', + 'Subject for certificate for users') + +flags.DEFINE_string('auth_driver', 'nova.auth.ldapdriver.FakeLdapDriver', + 'Driver that auth manager uses') + +class AuthBase(object): + """Base class for objects relating to auth + + Objects derived from this class should be stupid data objects with + an id member. They may optionally contain methods that delegate to + AuthManager, but should not implement logic themselves. + """ + @classmethod + def safe_id(cls, obj): + """Safe get object id + + This method will return the id of the object if the object + is of this class, otherwise it will return the original object. + This allows methods to accept objects or ids as paramaters. + + """ + if isinstance(obj, cls): + return obj.id + else: + return obj + + +# anthony - the User class has moved to nova.models +#class User(AuthBase): +# """Object representing a user""" +# def __init__(self, id, name, access, secret, admin): +# AuthBase.__init__(self) +# self.id = id +# self.name = name +# self.access = access +# self.secret = secret +# self.admin = admin +# +# def is_superuser(self): +# return AuthManager().is_superuser(self) +# +# def is_admin(self): +# return AuthManager().is_admin(self) +# +# def has_role(self, role): +# return AuthManager().has_role(self, role) +# +# def add_role(self, role): +# return AuthManager().add_role(self, role) +# +# def remove_role(self, role): +# return AuthManager().remove_role(self, role) +# +# def is_project_member(self, project): +# return AuthManager().is_project_member(self, project) +# +# def is_project_manager(self, project): +# return AuthManager().is_project_manager(self, project) +# +# def generate_key_pair(self, name): +# return AuthManager().generate_key_pair(self.id, name) +# +# def create_key_pair(self, name, public_key, fingerprint): +# return AuthManager().create_key_pair(self.id, +# name, +# public_key, +# fingerprint) +# +# def get_key_pair(self, name): +# return AuthManager().get_key_pair(self.id, name) +# +# def delete_key_pair(self, name): +# return AuthManager().delete_key_pair(self.id, name) +# +# def get_key_pairs(self): +# return AuthManager().get_key_pairs(self.id) +# +# def __repr__(self): +# return "User('%s', '%s', '%s', '%s', %s)" % (self.id, +# self.name, +# self.access, +# self.secret, +# self.admin) + + +class KeyPair(AuthBase): + """Represents an ssh key returned from the datastore + + Even though this object is named KeyPair, only the public key and + fingerprint is stored. The user's private key is not saved. + """ + def __init__(self, id, name, owner_id, public_key, fingerprint): + AuthBase.__init__(self) + self.id = id + self.name = name + self.owner_id = owner_id + self.public_key = public_key + self.fingerprint = fingerprint + + def __repr__(self): + return "KeyPair('%s', '%s', '%s', '%s', '%s')" % (self.id, + self.name, + self.owner_id, + self.public_key, + self.fingerprint) + + +class Project(AuthBase): + """Represents a Project returned from the datastore""" + def __init__(self, id, name, project_manager_id, description, member_ids): + AuthBase.__init__(self) + self.id = id + self.name = name + self.project_manager_id = project_manager_id + self.description = description + self.member_ids = member_ids + + @property + def project_manager(self): + return AuthManager().get_user(self.project_manager_id) + + @property + def vpn_ip(self): + ip, port = AuthManager().get_project_vpn_data(self) + return ip + + @property + def vpn_port(self): + ip, port = AuthManager().get_project_vpn_data(self) + return port + + def has_manager(self, user): + return AuthManager().is_project_manager(user, self) + + def has_member(self, user): + return AuthManager().is_project_member(user, self) + + def add_role(self, user, role): + return AuthManager().add_role(user, role, self) + + def remove_role(self, user, role): + return AuthManager().remove_role(user, role, self) + + def has_role(self, user, role): + return AuthManager().has_role(user, role, self) + + def get_credentials(self, user): + return AuthManager().get_credentials(user, self) + + def __repr__(self): + return "Project('%s', '%s', '%s', '%s', %s)" % (self.id, + self.name, + self.project_manager_id, + self.description, + self.member_ids) + + + +class AuthManager(object): + """Manager Singleton for dealing with Users, Projects, and Keypairs + + Methods accept objects or ids. + + AuthManager uses a driver object to make requests to the data backend. + See ldapdriver for reference. + + AuthManager also manages associated data related to Auth objects that + need to be more accessible, such as vpn ips and ports. + """ + _instance = None + def __new__(cls, *args, **kwargs): + """Returns the AuthManager singleton""" + if not cls._instance: + cls._instance = super(AuthManager, cls).__new__(cls) + return cls._instance + + def __init__(self, driver=None, *args, **kwargs): + """Inits the driver from parameter or flag + + __init__ is run every time AuthManager() is called, so we only + reset the driver if it is not set or a new driver is specified. + """ + if driver or not getattr(self, 'driver', None): + self.driver = utils.import_class(driver or FLAGS.auth_driver) + + def authenticate(self, access, signature, params, verb='GET', + server_string='127.0.0.1:8773', path='/', + check_type='ec2', headers=None): + """Authenticates AWS request using access key and signature + + If the project is not specified, attempts to authenticate to + a project with the same name as the user. This way, older tools + that have no project knowledge will still work. + + @type access: str + @param access: Access key for user in the form "access:project". + + @type signature: str + @param signature: Signature of the request. + + @type params: list of str + @param params: Web paramaters used for the signature. + + @type verb: str + @param verb: Web request verb ('GET' or 'POST'). + + @type server_string: str + @param server_string: Web request server string. + + @type path: str + @param path: Web request path. + + @type check_type: str + @param check_type: Type of signature to check. 'ec2' for EC2, 's3' for + S3. Any other value will cause signature not to be + checked. + + @type headers: list + @param headers: HTTP headers passed with the request (only needed for + s3 signature checks) + + @rtype: tuple (User, Project) + @return: User and project that the request represents. + """ + # TODO(vish): check for valid timestamp + (access_key, sep, project_id) = access.partition(':') + + logging.info('Looking up user: %r', access_key) + user = self.get_user_from_access_key(access_key) + logging.info('user: %r', user) + if user == None: + raise exception.NotFound('No user found for access key %s' % + access_key) + + # NOTE(vish): if we stop using project name as id we need better + # logic to find a default project for user + if project_id is '': + project_id = user.name + + project = self.get_project(project_id) + if project == None: + raise exception.NotFound('No project called %s could be found' % + project_id) + if not self.is_admin(user) and not self.is_project_member(user, + project): + raise exception.NotFound('User %s is not a member of project %s' % + (user.id, project.id)) + if check_type == 's3': + expected_signature = signer.Signer(user.secret.encode()).s3_authorization(headers, verb, path) + logging.debug('user.secret: %s', user.secret) + logging.debug('expected_signature: %s', expected_signature) + logging.debug('signature: %s', signature) + if signature != expected_signature: + raise exception.NotAuthorized('Signature does not match') + elif check_type == 'ec2': + # NOTE(vish): hmac can't handle unicode, so encode ensures that + # secret isn't unicode + expected_signature = signer.Signer(user.secret.encode()).generate( + params, verb, server_string, path) + logging.debug('user.secret: %s', user.secret) + logging.debug('expected_signature: %s', expected_signature) + logging.debug('signature: %s', signature) + if signature != expected_signature: + raise exception.NotAuthorized('Signature does not match') + return (user, project) + + def get_access_key(self, user, project): + """Get an access key that includes user and project""" + if not isinstance(user, User): + user = self.get_user(user) + return "%s:%s" % (user.access, Project.safe_id(project)) + + def is_superuser(self, user): + """Checks for superuser status, allowing user to bypass rbac + + @type user: User or uid + @param user: User to check. + + @rtype: bool + @return: True for superuser. + """ + if not isinstance(user, User): + user = self.get_user(user) + # NOTE(vish): admin flag on user represents superuser + if user.admin: + return True + for role in FLAGS.superuser_roles: + if self.has_role(user, role): + return True + + def is_admin(self, user): + """Checks for admin status, allowing user to access all projects + + @type user: User or uid + @param user: User to check. + + @rtype: bool + @return: True for admin. + """ + if not isinstance(user, User): + user = self.get_user(user) + if self.is_superuser(user): + return True + for role in FLAGS.global_roles: + if self.has_role(user, role): + return True + + def has_role(self, user, role, project=None): + """Checks existence of role for user + + If project is not specified, checks for a global role. If project + is specified, checks for the union of the global role and the + project role. + + Role 'projectmanager' only works for projects and simply checks to + see if the user is the project_manager of the specified project. It + is the same as calling is_project_manager(user, project). + + @type user: User or uid + @param user: User to check. + + @type role: str + @param role: Role to check. + + @type project: Project or project_id + @param project: Project in which to look for local role. + + @rtype: bool + @return: True if the user has the role. + """ + with self.driver() as drv: + if role == 'projectmanager': + if not project: + raise exception.Error("Must specify project") + return self.is_project_manager(user, project) + + global_role = drv.has_role(User.safe_id(user), + role, + None) + if not global_role: + return global_role + + if not project or role in FLAGS.global_roles: + return global_role + + return drv.has_role(User.safe_id(user), + role, + Project.safe_id(project)) + + def add_role(self, user, role, project=None): + """Adds role for user + + If project is not specified, adds a global role. If project + is specified, adds a local role. + + The 'projectmanager' role is special and can't be added or removed. + + @type user: User or uid + @param user: User to which to add role. + + @type role: str + @param role: Role to add. + + @type project: Project or project_id + @param project: Project in which to add local role. + """ + with self.driver() as drv: + drv.add_role(User.safe_id(user), role, Project.safe_id(project)) + + def remove_role(self, user, role, project=None): + """Removes role for user + + If project is not specified, removes a global role. If project + is specified, removes a local role. + + The 'projectmanager' role is special and can't be added or removed. + + @type user: User or uid + @param user: User from which to remove role. + + @type role: str + @param role: Role to remove. + + @type project: Project or project_id + @param project: Project in which to remove local role. + """ + with self.driver() as drv: + drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) + + def get_project(self, pid): + """Get project object by id""" + with self.driver() as drv: + project_dict = drv.get_project(pid) + if project_dict: + return Project(**project_dict) + + def get_projects(self, user=None): + """Retrieves list of projects, optionally filtered by user""" + with self.driver() as drv: + project_list = drv.get_projects(User.safe_id(user)) + if not project_list: + return [] + return [Project(**project_dict) for project_dict in project_list] + + def create_project(self, name, manager_user, + description=None, member_users=None): + """Create a project + + @type name: str + @param name: Name of the project to create. The name will also be + used as the project id. + + @type manager_user: User or uid + @param manager_user: This user will be the project manager. + + @type description: str + @param project: Description of the project. If no description is + specified, the name of the project will be used. + + @type member_users: list of User or uid + @param: Initial project members. The project manager will always be + added as a member, even if he isn't specified in this list. + + @rtype: Project + @return: The new project. + """ + if member_users: + member_users = [User.safe_id(u) for u in member_users] + with self.driver() as drv: + project_dict = drv.create_project(name, + User.safe_id(manager_user), + description, + member_users) + if project_dict: + return Project(**project_dict) + + def add_to_project(self, user, project): + """Add user to project""" + with self.driver() as drv: + return drv.add_to_project(User.safe_id(user), + Project.safe_id(project)) + + def is_project_manager(self, user, project): + """Checks if user is project manager""" + if not isinstance(project, Project): + project = self.get_project(project) + return User.safe_id(user) == project.project_manager_id + + def is_project_member(self, user, project): + """Checks to see if user is a member of project""" + if not isinstance(project, Project): + project = self.get_project(project) + return User.safe_id(user) in project.member_ids + + def remove_from_project(self, user, project): + """Removes a user from a project""" + with self.driver() as drv: + return drv.remove_from_project(User.safe_id(user), + Project.safe_id(project)) + + def get_project_vpn_data(self, project): + """Gets vpn ip and port for project + + @type project: Project or project_id + @param project: Project from which to get associated vpn data + + @rvalue: tuple of (str, str) + @return: A tuple containing (ip, port) or None, None if vpn has + not been allocated for user. + """ + network_data = vpn.NetworkData.lookup(Project.safe_id(project)) + if not network_data: + raise exception.NotFound('project network data has not been set') + return (network_data.ip, network_data.port) + + def delete_project(self, project): + """Deletes a project""" + with self.driver() as drv: + return drv.delete_project(Project.safe_id(project)) + + def get_user(self, uid): + """Retrieves a user by id""" + with self.driver() as drv: + user_dict = drv.get_user(uid) + if user_dict: + return User(**user_dict) + + def get_user_from_access_key(self, access_key): + """Retrieves a user by access key""" + with self.driver() as drv: + user_dict = drv.get_user_from_access_key(access_key) + if user_dict: + return User(**user_dict) + + def get_users(self): + """Retrieves a list of all users""" + with self.driver() as drv: + user_list = drv.get_users() + if not user_list: + return [] + return [User(**user_dict) for user_dict in user_list] + + def create_user(self, name, access=None, secret=None, admin=False): + """Creates a user + + @type name: str + @param name: Name of the user to create. + + @type access: str + @param access: Access Key (defaults to a random uuid) + + @type secret: str + @param secret: Secret Key (defaults to a random uuid) + + @type admin: bool + @param admin: Whether to set the admin flag. The admin flag gives + superuser status regardless of roles specifed for the user. + + @type create_project: bool + @param: Whether to create a project for the user with the same name. + + @rtype: User + @return: The new user. + """ + if access == None: access = str(uuid.uuid4()) + if secret == None: secret = str(uuid.uuid4()) + with self.driver() as drv: + user_dict = drv.create_user(name, access, secret, admin) + if user_dict: + return User(**user_dict) + + def delete_user(self, user): + """Deletes a user""" + with self.driver() as drv: + drv.delete_user(User.safe_id(user)) + + def generate_key_pair(self, user, key_name): + """Generates a key pair for a user + + Generates a public and private key, stores the public key using the + key_name, and returns the private key and fingerprint. + + @type user: User or uid + @param user: User for which to create key pair. + + @type key_name: str + @param key_name: Name to use for the generated KeyPair. + + @rtype: tuple (private_key, fingerprint) + @return: A tuple containing the private_key and fingerprint. + """ + # NOTE(vish): generating key pair is slow so check for legal + # creation before creating keypair + uid = User.safe_id(user) + with self.driver() as drv: + if not drv.get_user(uid): + raise exception.NotFound("User %s doesn't exist" % user) + if drv.get_key_pair(uid, key_name): + raise exception.Duplicate("The keypair %s already exists" + % key_name) + private_key, public_key, fingerprint = crypto.generate_key_pair() + self.create_key_pair(uid, key_name, public_key, fingerprint) + return private_key, fingerprint + + def create_key_pair(self, user, key_name, public_key, fingerprint): + """Creates a key pair for user""" + with self.driver() as drv: + kp_dict = drv.create_key_pair(User.safe_id(user), + key_name, + public_key, + fingerprint) + if kp_dict: + return KeyPair(**kp_dict) + + def get_key_pair(self, user, key_name): + """Retrieves a key pair for user""" + with self.driver() as drv: + kp_dict = drv.get_key_pair(User.safe_id(user), key_name) + if kp_dict: + return KeyPair(**kp_dict) + + def get_key_pairs(self, user): + """Retrieves all key pairs for user""" + with self.driver() as drv: + kp_list = drv.get_key_pairs(User.safe_id(user)) + if not kp_list: + return [] + return [KeyPair(**kp_dict) for kp_dict in kp_list] + + def delete_key_pair(self, user, key_name): + """Deletes a key pair for user""" + with self.driver() as drv: + drv.delete_key_pair(User.safe_id(user), key_name) + + def get_credentials(self, user, project=None): + """Get credential zip for user in project""" + if not isinstance(user, User): + user = self.get_user(user) + if project is None: + project = user.id + pid = Project.safe_id(project) + rc = self.__generate_rc(user.access, user.secret, pid) + private_key, signed_cert = self._generate_x509_cert(user.id, pid) + + tmpdir = tempfile.mkdtemp() + zf = os.path.join(tmpdir, "temp.zip") + zippy = zipfile.ZipFile(zf, 'w') + zippy.writestr(FLAGS.credential_rc_file, rc) + zippy.writestr(FLAGS.credential_key_file, private_key) + zippy.writestr(FLAGS.credential_cert_file, signed_cert) + + network_data = vpn.NetworkData.lookup(pid) + if network_data: + configfile = open(FLAGS.vpn_client_template,"r") + s = string.Template(configfile.read()) + configfile.close() + config = s.substitute(keyfile=FLAGS.credential_key_file, + certfile=FLAGS.credential_cert_file, + ip=network_data.ip, + port=network_data.port) + zippy.writestr(FLAGS.credential_vpn_file, config) + else: + logging.warn("No vpn data for project %s" % + pid) + + zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(user.id)) + zippy.close() + with open(zf, 'rb') as f: + buffer = f.read() + + shutil.rmtree(tmpdir) + return buffer + + def get_environment_rc(self, user, project=None): + """Get credential zip for user in project""" + if not isinstance(user, User): + user = self.get_user(user) + if project is None: + project = user.id + pid = Project.safe_id(project) + return self.__generate_rc(user.access, user.secret, pid) + + def __generate_rc(self, access, secret, pid): + """Generate rc file for user""" + rc = open(FLAGS.credentials_template).read() + rc = rc % { 'access': access, + 'project': pid, + 'secret': secret, + 'ec2': FLAGS.ec2_url, + 's3': 'http://%s:%s' % (FLAGS.s3_host, FLAGS.s3_port), + 'nova': FLAGS.ca_file, + 'cert': FLAGS.credential_cert_file, + 'key': FLAGS.credential_key_file, + } + return rc + + def _generate_x509_cert(self, uid, pid): + """Generate x509 cert for user""" + (private_key, csr) = crypto.generate_x509_cert( + self.__cert_subject(uid)) + # TODO(joshua): This should be async call back to the cloud controller + signed_cert = crypto.sign_csr(csr, pid) + return (private_key, signed_cert) + + def __cert_subject(self, uid): + """Helper to generate cert subject""" + return FLAGS.credential_cert_subject % (uid, utils.isotime()) diff --git a/nova/models.py b/nova/models.py new file mode 100644 index 00000000..4c739488 --- /dev/null +++ b/nova/models.py @@ -0,0 +1,198 @@ +from sqlalchemy.orm import relationship, backref, validates +from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime, Boolean, Text +from sqlalchemy.ext.declarative import declarative_base +from auth import * + +Base = declarative_base() + +class User(Base): + # sqlalchemy + __tablename__ = 'users' + sid = Column(String, primary_key=True) + + # backwards compatibility + @classmethod + def safe_id(cls, obj): + """Safe get object id + + This method will return the id of the object if the object + is of this class, otherwise it will return the original object. + This allows methods to accept objects or ids as paramaters. + + """ + if isinstance(obj, cls): + return obj.id + else: + return obj + +# def __init__(self, id, name, access, secret, admin): +# self.id = id +# self.name = name +# self.access = access +# self.secret = secret +# self.admin = admin + + def __getattr__(self, name): + if name == 'id': + return self.uid + else: raise AttributeError, name + + def is_superuser(self): + return AuthManager().is_superuser(self) + + def is_admin(self): + return AuthManager().is_admin(self) + + def has_role(self, role): + return AuthManager().has_role(self, role) + + def add_role(self, role): + return AuthManager().add_role(self, role) + + def remove_role(self, role): + return AuthManager().remove_role(self, role) + + def is_project_member(self, project): + return AuthManager().is_project_member(self, project) + + def is_project_manager(self, project): + return AuthManager().is_project_manager(self, project) + + def generate_key_pair(self, name): + return AuthManager().generate_key_pair(self.id, name) + + def create_key_pair(self, name, public_key, fingerprint): + return AuthManager().create_key_pair(self.id, + name, + public_key, + fingerprint) + + def get_key_pair(self, name): + return AuthManager().get_key_pair(self.id, name) + + def delete_key_pair(self, name): + return AuthManager().delete_key_pair(self.id, name) + + def get_key_pairs(self): + return AuthManager().get_key_pairs(self.id) + + def __repr__(self): + return "User('%s', '%s', '%s', '%s', %s)" % (self.id, + self.name, + self.access, + self.secret, + self.admin) + + + +class Project(Base): + __tablename__ = 'projects' + sid = Column(String, primary_key=True) + +class Image(Base): + __tablename__ = 'images' + user_sid = Column(String, ForeignKey('users.sid'), nullable=False) + project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + + sid = Column(String, primary_key=True) + image_type = Column(String) + public = Column(Boolean, default=False) + state = Column(String) + location = Column(String) + arch = Column(String) + default_kernel_sid = Column(String) + default_ramdisk_sid = Column(String) + + created_at = Column(DateTime) + updated_at = Column(DateTime) # auto update on change FIXME + + + @validates('image_type') + def validate_image_type(self, key, image_type): + assert(image_type in ['machine', 'kernel', 'ramdisk', 'raw']) + + @validates('state') + def validate_state(self, key, state): + assert(state in ['available', 'pending', 'disabled']) + + @validates('default_kernel_sid') + def validate_kernel_sid(self, key, val): + if val != 'machine': + assert(val is None) + + @validates('default_ramdisk_sid') + def validate_ramdisk_sid(self, key, val): + if val != 'machine': + assert(val is None) + +class Network(Base): + __tablename__ = 'networks' + id = Column(Integer, primary_key=True) + bridge = Column(String) + vlan = Column(String) + #vpn_port = Column(Integer) + project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + +class PhysicalNode(Base): + __tablename__ = 'physical_nodes' + id = Column(Integer, primary_key=True) + +class Instance(Base): + __tablename__ = 'instances' + id = Column(Integer, primary_key=True) + + user_sid = Column(String, ForeignKey('users.sid'), nullable=False) + project_sid = Column(String, ForeignKey('projects.sid')) + + image_sid = Column(Integer, ForeignKey('images.sid'), nullable=False) + kernel_sid = Column(String, ForeignKey('images.sid'), nullable=True) + ramdisk_sid = Column(String, ForeignKey('images.sid'), nullable=True) + + launch_index = Column(Integer) + key_name = Column(String) + key_data = Column(Text) + + state = Column(String) + + hostname = Column(String) + physical_node_id = Column(Integer) + + instance_type = Column(Integer) + + user_data = Column(Text) + +# user = relationship(User, backref=backref('instances', order_by=id)) +# ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) +# kernel = relationship(Kernel, backref=backref('instances', order_by=id)) +# project = relationship(Project, backref=backref('instances', order_by=id)) + +#TODO - see Ewan's email about state improvements + # vmstate_state = running, halted, suspended, paused + # power_state = what we have + # task_state = transitory and may trigger power state transition + + @validates('state') + def validate_state(self, key, state): + assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) + +class Volume(Base): + __tablename__ = 'volumes' + id = Column(Integer, primary_key=True) + shelf_id = Column(Integer) + blade_id = Column(Integer) + + +if __name__ == '__main__': + from sqlalchemy import create_engine + engine = create_engine('sqlite:///:memory:', echo=True) + Base.metadata.create_all(engine) + + from sqlalchemy.orm import sessionmaker + Session = sessionmaker(bind=engine) + session = Session() + + instance = Instance(image_sid='as', ramdisk_sid='AS', user_sid='anthony') + user = User(sid='anthony') + session.add(instance) + session.commit() + From 8675502fe1d5d6cdf6d624f1580c427b44a19c91 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 14 Aug 2010 18:04:43 -0700 Subject: [PATCH 003/113] got run_tests.py to run (with many failed tests) --- nova/datastore.old.py | 261 +++++++++++++++++++++++++++++++++++++++++ nova/datastore.py | 262 ------------------------------------------ run_tests.py | 10 +- 3 files changed, 266 insertions(+), 267 deletions(-) create mode 100644 nova/datastore.old.py diff --git a/nova/datastore.old.py b/nova/datastore.old.py new file mode 100644 index 00000000..751c5eee --- /dev/null +++ b/nova/datastore.old.py @@ -0,0 +1,261 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Datastore: + +MAKE Sure that ReDIS is running, and your flags are set properly, +before trying to run this. +""" + +import logging + +from nova import exception +from nova import flags +from nova import utils + + +FLAGS = flags.FLAGS +flags.DEFINE_string('redis_host', '127.0.0.1', + 'Host that redis is running on.') +flags.DEFINE_integer('redis_port', 6379, + 'Port that redis is running on.') +flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') + + +class Redis(object): + def __init__(self): + if hasattr(self.__class__, '_instance'): + raise Exception('Attempted to instantiate singleton') + + @classmethod + def instance(cls): + if not hasattr(cls, '_instance'): + inst = redis.Redis(host=FLAGS.redis_host, + port=FLAGS.redis_port, + db=FLAGS.redis_db) + cls._instance = inst + return cls._instance + + +class ConnectionError(exception.Error): + pass + + +def absorb_connection_error(fn): + def _wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except redis.exceptions.ConnectionError, ce: + raise ConnectionError(str(ce)) + return _wrapper + + +class BasicModel(object): + """ + All Redis-backed data derives from this class. + + You MUST specify an identifier() property that returns a unique string + per instance. + + You MUST have an initializer that takes a single argument that is a value + returned by identifier() to load a new class with. + + You may want to specify a dictionary for default_state(). + + You may also specify override_type at the class left to use a key other + than __class__.__name__. + + You override save and destroy calls to automatically build and destroy + associations. + """ + + override_type = None + + @absorb_connection_error + def __init__(self): + state = Redis.instance().hgetall(self.__redis_key) + if state: + self.initial_state = state + self.state = dict(self.initial_state) + else: + self.initial_state = {} + self.state = self.default_state() + + + def default_state(self): + """You probably want to define this in your subclass""" + return {} + + @classmethod + def _redis_name(cls): + return cls.override_type or cls.__name__.lower() + + @classmethod + def lookup(cls, identifier): + rv = cls(identifier) + if rv.is_new_record(): + return None + else: + return rv + + @classmethod + @absorb_connection_error + def all(cls): + """yields all objects in the store""" + redis_set = cls._redis_set_name(cls.__name__) + for identifier in Redis.instance().smembers(redis_set): + yield cls(identifier) + + @classmethod + def associated_to(cls, foreign_type, foreign_id): + for identifier in cls.associated_keys(foreign_type, foreign_id): + yield cls(identifier) + + @classmethod + @absorb_connection_error + def associated_keys(cls, foreign_type, foreign_id): + redis_set = cls._redis_association_name(foreign_type, foreign_id) + return Redis.instance().smembers(redis_set) or [] + + @classmethod + def _redis_set_name(cls, kls_name): + # stupidly pluralize (for compatiblity with previous codebase) + return kls_name.lower() + "s" + + @classmethod + def _redis_association_name(cls, foreign_type, foreign_id): + return cls._redis_set_name("%s:%s:%s" % + (foreign_type, foreign_id, cls._redis_name())) + + @property + def identifier(self): + """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define identifier") + + @property + def __redis_key(self): + return '%s:%s' % (self._redis_name(), self.identifier) + + def __repr__(self): + return "<%s:%s>" % (self.__class__.__name__, self.identifier) + + def keys(self): + return self.state.keys() + + def copy(self): + copyDict = {} + for item in self.keys(): + copyDict[item] = self[item] + return copyDict + + def get(self, item, default): + return self.state.get(item, default) + + def update(self, update_dict): + return self.state.update(update_dict) + + def setdefault(self, item, default): + return self.state.setdefault(item, default) + + def __contains__(self, item): + return item in self.state + + def __getitem__(self, item): + return self.state[item] + + def __setitem__(self, item, val): + self.state[item] = val + return self.state[item] + + def __delitem__(self, item): + """We don't support this""" + raise Exception("Silly monkey, models NEED all their properties.") + + def is_new_record(self): + return self.initial_state == {} + + @absorb_connection_error + def add_to_index(self): + """Each insance of Foo has its id tracked int the set named Foos""" + set_name = self.__class__._redis_set_name(self.__class__.__name__) + Redis.instance().sadd(set_name, self.identifier) + + @absorb_connection_error + def remove_from_index(self): + """Remove id of this instance from the set tracking ids of this type""" + set_name = self.__class__._redis_set_name(self.__class__.__name__) + Redis.instance().srem(set_name, self.identifier) + + @absorb_connection_error + def associate_with(self, foreign_type, foreign_id): + """Add this class id into the set foreign_type:foreign_id:this_types""" + # note the extra 's' on the end is for plurality + # to match the old data without requiring a migration of any sort + self.add_associated_model_to_its_set(foreign_type, foreign_id) + redis_set = self.__class__._redis_association_name(foreign_type, + foreign_id) + Redis.instance().sadd(redis_set, self.identifier) + + @absorb_connection_error + def unassociate_with(self, foreign_type, foreign_id): + """Delete from foreign_type:foreign_id:this_types set""" + redis_set = self.__class__._redis_association_name(foreign_type, + foreign_id) + Redis.instance().srem(redis_set, self.identifier) + + def add_associated_model_to_its_set(self, model_type, model_id): + """ + When associating an X to a Y, save Y for newer timestamp, etc, and to + make sure to save it if Y is a new record. + If the model_type isn't found as a usable class, ignore it, this can + happen when associating to things stored in LDAP (user, project, ...). + """ + table = globals() + klsname = model_type.capitalize() + if table.has_key(klsname): + model_class = table[klsname] + model_inst = model_class(model_id) + model_inst.save() + + @absorb_connection_error + def save(self): + """ + update the directory with the state from this model + also add it to the index of items of the same type + then set the initial_state = state so new changes are tracked + """ + # TODO(ja): implement hmset in redis-py and use it + # instead of multiple calls to hset + if self.is_new_record(): + self["create_time"] = utils.isotime() + for key, val in self.state.iteritems(): + Redis.instance().hset(self.__redis_key, key, val) + self.add_to_index() + self.initial_state = dict(self.state) + return True + + @absorb_connection_error + def destroy(self): + """deletes all related records from datastore.""" + logging.info("Destroying datamodel for %s %s", + self.__class__.__name__, self.identifier) + Redis.instance().delete(self.__redis_key) + self.remove_from_index() + return True + diff --git a/nova/datastore.py b/nova/datastore.py index 5dc6ed10..e69de29b 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -1,262 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Datastore: - -MAKE Sure that ReDIS is running, and your flags are set properly, -before trying to run this. -""" - -import logging -import redis - -from nova import exception -from nova import flags -from nova import utils - - -FLAGS = flags.FLAGS -flags.DEFINE_string('redis_host', '127.0.0.1', - 'Host that redis is running on.') -flags.DEFINE_integer('redis_port', 6379, - 'Port that redis is running on.') -flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') - - -class Redis(object): - def __init__(self): - if hasattr(self.__class__, '_instance'): - raise Exception('Attempted to instantiate singleton') - - @classmethod - def instance(cls): - if not hasattr(cls, '_instance'): - inst = redis.Redis(host=FLAGS.redis_host, - port=FLAGS.redis_port, - db=FLAGS.redis_db) - cls._instance = inst - return cls._instance - - -class ConnectionError(exception.Error): - pass - - -def absorb_connection_error(fn): - def _wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except redis.exceptions.ConnectionError, ce: - raise ConnectionError(str(ce)) - return _wrapper - - -class BasicModel(object): - """ - All Redis-backed data derives from this class. - - You MUST specify an identifier() property that returns a unique string - per instance. - - You MUST have an initializer that takes a single argument that is a value - returned by identifier() to load a new class with. - - You may want to specify a dictionary for default_state(). - - You may also specify override_type at the class left to use a key other - than __class__.__name__. - - You override save and destroy calls to automatically build and destroy - associations. - """ - - override_type = None - - @absorb_connection_error - def __init__(self): - state = Redis.instance().hgetall(self.__redis_key) - if state: - self.initial_state = state - self.state = dict(self.initial_state) - else: - self.initial_state = {} - self.state = self.default_state() - - - def default_state(self): - """You probably want to define this in your subclass""" - return {} - - @classmethod - def _redis_name(cls): - return cls.override_type or cls.__name__.lower() - - @classmethod - def lookup(cls, identifier): - rv = cls(identifier) - if rv.is_new_record(): - return None - else: - return rv - - @classmethod - @absorb_connection_error - def all(cls): - """yields all objects in the store""" - redis_set = cls._redis_set_name(cls.__name__) - for identifier in Redis.instance().smembers(redis_set): - yield cls(identifier) - - @classmethod - def associated_to(cls, foreign_type, foreign_id): - for identifier in cls.associated_keys(foreign_type, foreign_id): - yield cls(identifier) - - @classmethod - @absorb_connection_error - def associated_keys(cls, foreign_type, foreign_id): - redis_set = cls._redis_association_name(foreign_type, foreign_id) - return Redis.instance().smembers(redis_set) or [] - - @classmethod - def _redis_set_name(cls, kls_name): - # stupidly pluralize (for compatiblity with previous codebase) - return kls_name.lower() + "s" - - @classmethod - def _redis_association_name(cls, foreign_type, foreign_id): - return cls._redis_set_name("%s:%s:%s" % - (foreign_type, foreign_id, cls._redis_name())) - - @property - def identifier(self): - """You DEFINITELY want to define this in your subclass""" - raise NotImplementedError("Your subclass should define identifier") - - @property - def __redis_key(self): - return '%s:%s' % (self._redis_name(), self.identifier) - - def __repr__(self): - return "<%s:%s>" % (self.__class__.__name__, self.identifier) - - def keys(self): - return self.state.keys() - - def copy(self): - copyDict = {} - for item in self.keys(): - copyDict[item] = self[item] - return copyDict - - def get(self, item, default): - return self.state.get(item, default) - - def update(self, update_dict): - return self.state.update(update_dict) - - def setdefault(self, item, default): - return self.state.setdefault(item, default) - - def __contains__(self, item): - return item in self.state - - def __getitem__(self, item): - return self.state[item] - - def __setitem__(self, item, val): - self.state[item] = val - return self.state[item] - - def __delitem__(self, item): - """We don't support this""" - raise Exception("Silly monkey, models NEED all their properties.") - - def is_new_record(self): - return self.initial_state == {} - - @absorb_connection_error - def add_to_index(self): - """Each insance of Foo has its id tracked int the set named Foos""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().sadd(set_name, self.identifier) - - @absorb_connection_error - def remove_from_index(self): - """Remove id of this instance from the set tracking ids of this type""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().srem(set_name, self.identifier) - - @absorb_connection_error - def associate_with(self, foreign_type, foreign_id): - """Add this class id into the set foreign_type:foreign_id:this_types""" - # note the extra 's' on the end is for plurality - # to match the old data without requiring a migration of any sort - self.add_associated_model_to_its_set(foreign_type, foreign_id) - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().sadd(redis_set, self.identifier) - - @absorb_connection_error - def unassociate_with(self, foreign_type, foreign_id): - """Delete from foreign_type:foreign_id:this_types set""" - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().srem(redis_set, self.identifier) - - def add_associated_model_to_its_set(self, model_type, model_id): - """ - When associating an X to a Y, save Y for newer timestamp, etc, and to - make sure to save it if Y is a new record. - If the model_type isn't found as a usable class, ignore it, this can - happen when associating to things stored in LDAP (user, project, ...). - """ - table = globals() - klsname = model_type.capitalize() - if table.has_key(klsname): - model_class = table[klsname] - model_inst = model_class(model_id) - model_inst.save() - - @absorb_connection_error - def save(self): - """ - update the directory with the state from this model - also add it to the index of items of the same type - then set the initial_state = state so new changes are tracked - """ - # TODO(ja): implement hmset in redis-py and use it - # instead of multiple calls to hset - if self.is_new_record(): - self["create_time"] = utils.isotime() - for key, val in self.state.iteritems(): - Redis.instance().hset(self.__redis_key, key, val) - self.add_to_index() - self.initial_state = dict(self.state) - return True - - @absorb_connection_error - def destroy(self): - """deletes all related records from datastore.""" - logging.info("Destroying datamodel for %s %s", - self.__class__.__name__, self.identifier) - Redis.instance().delete(self.__redis_key) - self.remove_from_index() - return True - diff --git a/run_tests.py b/run_tests.py index d90ac817..f0a5efb7 100644 --- a/run_tests.py +++ b/run_tests.py @@ -84,11 +84,11 @@ if __name__ == '__main__': if FLAGS.fake_tests: from nova.tests.fake_flags import * # use db 8 for fake tests - FLAGS.redis_db = 8 - if FLAGS.flush_db: - logging.info("Flushing redis datastore") - r = datastore.Redis.instance() - r.flushdb() + #FLAGS.redis_db = 8 + #if FLAGS.flush_db: + # logging.info("Flushing redis datastore") + # r = datastore.Redis.instance() + # r.flushdb() else: from nova.tests.real_flags import * From 8ea845108d169b1edbdadf53bc4c3b017d7b5fb8 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 14 Aug 2010 18:31:23 -0700 Subject: [PATCH 004/113] make the fake-ldap system work again --- nova/datastore.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/nova/datastore.py b/nova/datastore.py index e69de29b..8e251942 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Datastore: + +MAKE Sure that ReDIS is running, and your flags are set properly, +before trying to run this. +""" + +import logging +import redis + +from nova import flags + +FLAGS = flags.FLAGS +flags.DEFINE_string('redis_host', '127.0.0.1', + 'Host that redis is running on.') +flags.DEFINE_integer('redis_port', 6379, + 'Port that redis is running on.') +flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') + + +class Redis(object): + def __init__(self): + if hasattr(self.__class__, '_instance'): + raise Exception('Attempted to instantiate singleton') + + @classmethod + def instance(cls): + if not hasattr(cls, '_instance'): + inst = redis.Redis(host=FLAGS.redis_host, + port=FLAGS.redis_port, + db=FLAGS.redis_db) + cls._instance = inst + return cls._instance + + From 62798f2a22814c1e1807a1f0a8c7c81e75ac862a Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 14 Aug 2010 18:39:00 -0700 Subject: [PATCH 005/113] re-add redis clearing --- run_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/run_tests.py b/run_tests.py index f0a5efb7..d90ac817 100644 --- a/run_tests.py +++ b/run_tests.py @@ -84,11 +84,11 @@ if __name__ == '__main__': if FLAGS.fake_tests: from nova.tests.fake_flags import * # use db 8 for fake tests - #FLAGS.redis_db = 8 - #if FLAGS.flush_db: - # logging.info("Flushing redis datastore") - # r = datastore.Redis.instance() - # r.flushdb() + FLAGS.redis_db = 8 + if FLAGS.flush_db: + logging.info("Flushing redis datastore") + r = datastore.Redis.instance() + r.flushdb() else: from nova.tests.real_flags import * From ff5c83325cf87680db9574917b5d40250edaa3a9 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 14 Aug 2010 21:24:26 -0700 Subject: [PATCH 006/113] more work on trying to get compute tests passing --- nova/auth/manager.py | 10 ++- nova/models.py | 148 ++++++++++----------------------- nova/tests/compute_unittest.py | 37 ++++++--- 3 files changed, 75 insertions(+), 120 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 064fd78b..f7f45489 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -31,6 +31,7 @@ import zipfile from nova import crypto from nova import exception from nova import flags +from nova import models from nova import utils from nova.auth import signer from nova.network import vpn @@ -201,6 +202,11 @@ class Project(AuthBase): ip, port = AuthManager().get_project_vpn_data(self) return port + @property + def network(self): + session = models.create_session() + return session.query(models.Network).filter_by(project_id=self.id).first() + def has_manager(self, user): return AuthManager().is_project_manager(user, self) @@ -521,7 +527,9 @@ class AuthManager(object): description, member_users) if project_dict: - return Project(**project_dict) + project = Project(**project_dict) + # FIXME(ja): create network? + return project def add_to_project(self, user, project): """Add user to project""" diff --git a/nova/models.py b/nova/models.py index 4c739488..06761602 100644 --- a/nova/models.py +++ b/nova/models.py @@ -1,107 +1,23 @@ from sqlalchemy.orm import relationship, backref, validates from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime, Boolean, Text from sqlalchemy.ext.declarative import declarative_base -from auth import * +from nova import auth Base = declarative_base() -class User(Base): - # sqlalchemy - __tablename__ = 'users' - sid = Column(String, primary_key=True) - - # backwards compatibility - @classmethod - def safe_id(cls, obj): - """Safe get object id - - This method will return the id of the object if the object - is of this class, otherwise it will return the original object. - This allows methods to accept objects or ids as paramaters. - - """ - if isinstance(obj, cls): - return obj.id - else: - return obj - -# def __init__(self, id, name, access, secret, admin): -# self.id = id -# self.name = name -# self.access = access -# self.secret = secret -# self.admin = admin - - def __getattr__(self, name): - if name == 'id': - return self.uid - else: raise AttributeError, name - - def is_superuser(self): - return AuthManager().is_superuser(self) - - def is_admin(self): - return AuthManager().is_admin(self) - - def has_role(self, role): - return AuthManager().has_role(self, role) - - def add_role(self, role): - return AuthManager().add_role(self, role) - - def remove_role(self, role): - return AuthManager().remove_role(self, role) - - def is_project_member(self, project): - return AuthManager().is_project_member(self, project) - - def is_project_manager(self, project): - return AuthManager().is_project_manager(self, project) - - def generate_key_pair(self, name): - return AuthManager().generate_key_pair(self.id, name) - - def create_key_pair(self, name, public_key, fingerprint): - return AuthManager().create_key_pair(self.id, - name, - public_key, - fingerprint) - - def get_key_pair(self, name): - return AuthManager().get_key_pair(self.id, name) - - def delete_key_pair(self, name): - return AuthManager().delete_key_pair(self.id, name) - - def get_key_pairs(self): - return AuthManager().get_key_pairs(self.id) - - def __repr__(self): - return "User('%s', '%s', '%s', '%s', %s)" % (self.id, - self.name, - self.access, - self.secret, - self.admin) - - - -class Project(Base): - __tablename__ = 'projects' - sid = Column(String, primary_key=True) - class Image(Base): __tablename__ = 'images' - user_sid = Column(String, ForeignKey('users.sid'), nullable=False) - project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + user_id = Column(String)#, ForeignKey('users.id'), nullable=False) + project_id = Column(String)#, ForeignKey('projects.id'), nullable=False) - sid = Column(String, primary_key=True) + id = Column(String, primary_key=True) image_type = Column(String) public = Column(Boolean, default=False) state = Column(String) location = Column(String) arch = Column(String) - default_kernel_sid = Column(String) - default_ramdisk_sid = Column(String) + default_kernel_id = Column(String) + default_ramdisk_id = Column(String) created_at = Column(DateTime) updated_at = Column(DateTime) # auto update on change FIXME @@ -115,13 +31,13 @@ class Image(Base): def validate_state(self, key, state): assert(state in ['available', 'pending', 'disabled']) - @validates('default_kernel_sid') - def validate_kernel_sid(self, key, val): + @validates('default_kernel_id') + def validate_kernel_id(self, key, val): if val != 'machine': assert(val is None) - @validates('default_ramdisk_sid') - def validate_ramdisk_sid(self, key, val): + @validates('default_ramdisk_id') + def validate_ramdisk_id(self, key, val): if val != 'machine': assert(val is None) @@ -131,7 +47,7 @@ class Network(Base): bridge = Column(String) vlan = Column(String) #vpn_port = Column(Integer) - project_sid = Column(String, ForeignKey('projects.sid'), nullable=False) + project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) class PhysicalNode(Base): __tablename__ = 'physical_nodes' @@ -141,16 +57,25 @@ class Instance(Base): __tablename__ = 'instances' id = Column(Integer, primary_key=True) - user_sid = Column(String, ForeignKey('users.sid'), nullable=False) - project_sid = Column(String, ForeignKey('projects.sid')) + user_id = Column(String) #, ForeignKey('users.id'), nullable=False) + project_id = Column(String) #, ForeignKey('projects.id')) - image_sid = Column(Integer, ForeignKey('images.sid'), nullable=False) - kernel_sid = Column(String, ForeignKey('images.sid'), nullable=True) - ramdisk_sid = Column(String, ForeignKey('images.sid'), nullable=True) + @property + def user(self): + return auth.manager.AuthManager().get_user(self.user_id) + + @property + def project(self): + return auth.manager.AuthManager().get_project(self.project_id) + + image_id = Column(Integer, ForeignKey('images.id'), nullable=False) + kernel_id = Column(String, ForeignKey('images.id'), nullable=True) + ramdisk_id = Column(String, ForeignKey('images.id'), nullable=True) launch_index = Column(Integer) key_name = Column(String) key_data = Column(Text) + security_group = Column(String) state = Column(String) @@ -161,7 +86,6 @@ class Instance(Base): user_data = Column(Text) -# user = relationship(User, backref=backref('instances', order_by=id)) # ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) # kernel = relationship(Kernel, backref=backref('instances', order_by=id)) # project = relationship(Project, backref=backref('instances', order_by=id)) @@ -182,17 +106,29 @@ class Volume(Base): blade_id = Column(Integer) -if __name__ == '__main__': +engine = None +def create_engine(): + global engine + if engine is not None: + return engine from sqlalchemy import create_engine engine = create_engine('sqlite:///:memory:', echo=True) - Base.metadata.create_all(engine) + Base.metadata.create_all(engine) + return engine +def create_session(engine=None): + if engine is None: + engine = create_engine() from sqlalchemy.orm import sessionmaker Session = sessionmaker(bind=engine) - session = Session() + return Session() - instance = Instance(image_sid='as', ramdisk_sid='AS', user_sid='anthony') - user = User(sid='anthony') +if __name__ == '__main__': + engine = create_engine() + session = create_session(engine) + + instance = Instance(image_id='as', ramdisk_id='AS', user_id='anthony') + user = User(id='anthony') session.add(instance) session.commit() diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index da0f82e3..c079f9a4 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -25,7 +25,8 @@ from nova import exception from nova import flags from nova import test from nova import utils -from nova.compute import model +from nova import models +from nova.auth import manager from nova.compute import service @@ -60,21 +61,31 @@ class ComputeConnectionTestCase(test.TrialTestCase): self.flags(connection_type='fake', fake_storage=True) self.compute = service.ComputeService() + self.manager = manager.AuthManager() + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + + def tearDown(self): + self.manager.delete_user('fake') + self.manager.delete_project('fake') def create_instance(self): - instdir = model.InstanceDirectory() - inst = instdir.new() + session = models.create_session() + + inst = models.Instance(user_id='fake', project_id='fake', image_id='ami-test') + session.add(inst) + session.commit() # TODO(ja): add ami, ari, aki, user_data - inst['reservation_id'] = 'r-fakeres' - inst['launch_time'] = '10' - inst['user_id'] = 'fake' - inst['project_id'] = 'fake' - inst['instance_type'] = 'm1.tiny' - inst['node_name'] = FLAGS.node_name - inst['mac_address'] = utils.generate_mac() - inst['ami_launch_index'] = 0 - inst.save() - return inst['instance_id'] + # inst['reservation_id'] = 'r-fakeres' + # inst['launch_time'] = '10' + #inst['user_id'] = 'fake' + #inst['project_id'] = 'fake' + #inst['instance_type'] = 'm1.tiny' + #inst['node_name'] = FLAGS.node_name + #inst['mac_address'] = utils.generate_mac() + #inst['ami_launch_index'] = 0 + #inst.save() + return inst.id @defer.inlineCallbacks def test_run_describe_terminate(self): From 8b76dc4f752b663f07a6432c0cec98b861f68a7e Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 14 Aug 2010 22:55:04 -0700 Subject: [PATCH 007/113] ComputeConnectionTestCase is almost working again --- nova/auth/manager.py | 6 +++++- nova/models.py | 18 +++++++++++----- nova/tests/compute_unittest.py | 38 +++++++++++++--------------------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index f7f45489..4a813c86 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -528,7 +528,11 @@ class AuthManager(object): member_users) if project_dict: project = Project(**project_dict) - # FIXME(ja): create network? + # FIXME(ja): EVIL HACK - this should poll from a pool + session = models.create_session() + net = models.Network(project_id=project.id, kind='vlan') + session.add(net) + session.commit() return project def add_to_project(self, user, project): diff --git a/nova/models.py b/nova/models.py index 06761602..51600bd2 100644 --- a/nova/models.py +++ b/nova/models.py @@ -22,7 +22,6 @@ class Image(Base): created_at = Column(DateTime) updated_at = Column(DateTime) # auto update on change FIXME - @validates('image_type') def validate_image_type(self, key, image_type): assert(image_type in ['machine', 'kernel', 'ramdisk', 'raw']) @@ -46,6 +45,7 @@ class Network(Base): id = Column(Integer, primary_key=True) bridge = Column(String) vlan = Column(String) + kind = Column(String) #vpn_port = Column(Integer) project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) @@ -77,7 +77,8 @@ class Instance(Base): key_data = Column(Text) security_group = Column(String) - state = Column(String) + state = Column(Integer) + state_description = Column(String) hostname = Column(String) physical_node_id = Column(Integer) @@ -86,6 +87,13 @@ class Instance(Base): user_data = Column(Text) + def set_state(self, state_code, state_description=None): + from nova.compute import power_state + self.state = state_code + if not state_description: + state_description = power_state.name(state_code) + self.state_description = state_description + # ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) # kernel = relationship(Kernel, backref=backref('instances', order_by=id)) # project = relationship(Project, backref=backref('instances', order_by=id)) @@ -95,9 +103,9 @@ class Instance(Base): # power_state = what we have # task_state = transitory and may trigger power state transition - @validates('state') - def validate_state(self, key, state): - assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) + #@validates('state') + #def validate_state(self, key, state): + # assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) class Volume(Base): __tablename__ = 'volumes' diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index c079f9a4..b2a89a34 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -91,31 +91,25 @@ class ComputeConnectionTestCase(test.TrialTestCase): def test_run_describe_terminate(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) + yield self.compute.run_instance(instance_id) - rv = yield self.compute.describe_instances() - logging.info("Running instances: %s", rv) - self.assertEqual(rv[instance_id].name, instance_id) + session = models.create_session() + instances = session.query(models.Instance).all() + logging.info("Running instances: %s", instances) + self.assertEqual(len(instances), 1) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.terminate_instance(instance_id) - rv = yield self.compute.describe_instances() - logging.info("After terminating instances: %s", rv) - self.assertEqual(rv, {}) + instances = session.query(models.Instance).all() + logging.info("After terminating instances: %s", instances) + self.assertEqual(len(instances), 0) @defer.inlineCallbacks def test_reboot(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - + yield self.compute.run_instance(instance_id) yield self.compute.reboot_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.terminate_instance(instance_id) @defer.inlineCallbacks def test_console_output(self): @@ -129,10 +123,6 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_instance_existing(self): instance_id = self.create_instance() - rv = yield self.compute.run_instance(instance_id) - - rv = yield self.compute.describe_instances() - self.assertEqual(rv[instance_id].name, instance_id) - - self.assertRaises(exception.Error, self.compute.run_instance, instance_id) - rv = yield self.compute.terminate_instance(instance_id) + yield self.compute.run_instance(instance_id) + self.assertFailure(self.compute.run_instance(instance_id), exception.Error) + yield self.compute.terminate_instance(instance_id) From c51da587492e284ce67ac3411e8bbcead3ae59a4 Mon Sep 17 00:00:00 2001 From: Sleepsonthefloor Date: Sat, 14 Aug 2010 07:08:34 -0700 Subject: [PATCH 008/113] remove more direct session interactions --- nova/models.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nova/models.py b/nova/models.py index 79273965..62341a24 100644 --- a/nova/models.py +++ b/nova/models.py @@ -33,6 +33,12 @@ class NovaBase(object): session = NovaBase.get_session() return session.query(cls).all() + @classmethod + def find(cls, obj_id): + session = NovaBase.get_session() + #print cls + return session.query(cls).filter_by(id=obj_id).one() + def save(self): session = NovaBase.get_session() session.add(self) @@ -144,15 +150,13 @@ class Volume(Base): blade_id = Column(Integer) -def create_engine(): - return NovaBase.get_engine(); def create_session(engine=None): return NovaBase.get_session() if __name__ == '__main__': - engine = create_engine() - session = create_session(engine) + engine = NovasBase.create_engine() + session = NovasBase.create_session(engine) instance = Instance(image_id='as', ramdisk_id='AS', user_id='anthony') user = User(id='anthony') From 1515bfafa3c9d9393df09389ad81f4af65836eed Mon Sep 17 00:00:00 2001 From: Sleepsonthefloor Date: Sun, 15 Aug 2010 13:36:01 -0700 Subject: [PATCH 009/113] add refresh on model --- nova/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/models.py b/nova/models.py index 9cbebca7..561a722f 100644 --- a/nova/models.py +++ b/nova/models.py @@ -53,6 +53,10 @@ class NovaBase(object): session.delete(self) session.flush() + def refresh(self): + session = NovaBase.get_session() + session.refresh(self) + class Image(Base, NovaBase): __tablename__ = 'images' user_id = Column(String)#, ForeignKey('users.id'), nullable=False) From af9a54975e4dd300fc0f93db95ef937c71051ee8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 15 Aug 2010 15:55:53 -0700 Subject: [PATCH 010/113] refactoring volume and some cleanup in model and compute --- nova/models.py | 38 ++++++++++++++++++++++------ nova/tests/volume_unittest.py | 47 ++++++++++++++--------------------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/nova/models.py b/nova/models.py index 62341a24..c397270d 100644 --- a/nova/models.py +++ b/nova/models.py @@ -1,7 +1,8 @@ -from sqlalchemy.orm import relationship, backref, validates +from sqlalchemy.orm import relationship, backref, validates, exc from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime, Boolean, Text from sqlalchemy.ext.declarative import declarative_base from nova import auth +from nova import exception Base = declarative_base() @@ -14,9 +15,9 @@ class NovaBase(object): @classmethod def create_engine(cls): if NovaBase._engine is not None: - return _engine + return NovaBase._engine from sqlalchemy import create_engine - NovaBase._engine = create_engine('sqlite:///:memory:', echo=True) + NovaBase._engine = create_engine('sqlite:///:memory:', echo=False) Base.metadata.create_all(NovaBase._engine) return NovaBase._engine @@ -24,7 +25,7 @@ class NovaBase(object): def get_session(cls): from sqlalchemy.orm import sessionmaker if NovaBase._session == None: - NovaBase.create_engine(); + NovaBase.create_engine() NovaBase._session = sessionmaker(bind=NovaBase._engine)() return NovaBase._session @@ -37,13 +38,21 @@ class NovaBase(object): def find(cls, obj_id): session = NovaBase.get_session() #print cls - return session.query(cls).filter_by(id=obj_id).one() + try: + return session.query(cls).filter_by(id=obj_id).one() + except exc.NoResultFound: + raise exception.NotFound("No model for id %s" % obj_id) def save(self): session = NovaBase.get_session() session.add(self) session.commit() + def delete(self): + session = NovaBase.get_session() + session.delete(self) + session.flush() + class Image(Base, NovaBase): __tablename__ = 'images' user_id = Column(String)#, ForeignKey('users.id'), nullable=False) @@ -143,20 +152,33 @@ class Instance(Base, NovaBase): #def validate_state(self, key, state): # assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) -class Volume(Base): +class Volume(Base, NovaBase): __tablename__ = 'volumes' id = Column(Integer, primary_key=True) + volume_id = Column(String) shelf_id = Column(Integer) blade_id = Column(Integer) + user_id = Column(String) #, ForeignKey('users.id'), nullable=False) + project_id = Column(String) #, ForeignKey('projects.id')) + # FIXME: should be physical_node_id = Column(Integer) + node_name = Column(String) + size = Column(Integer) + alvailability_zone = Column(String) # FIXME foreign key? + instance_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + mountpoint = Column(String) + attach_time = Column(String) # FIXME datetime + status = Column(String) # FIXME enum? + attach_status = Column(String) # FIXME enum + delete_on_termination = Column(Boolean) def create_session(engine=None): return NovaBase.get_session() if __name__ == '__main__': - engine = NovasBase.create_engine() - session = NovasBase.create_session(engine) + engine = NovaBase.create_engine() + session = NovaBase.create_session(engine) instance = Instance(image_id='as', ramdisk_id='AS', user_id='anthony') user = User(id='anthony') diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 2a07afe6..e979995f 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -17,15 +17,14 @@ # under the License. import logging -import shutil -import tempfile from twisted.internet import defer -from nova import compute from nova import exception from nova import flags +from nova import models from nova import test +from nova.compute import service as compute_service from nova.volume import service as volume_service @@ -36,29 +35,22 @@ class VolumeTestCase(test.TrialTestCase): def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() - self.compute = compute.service.ComputeService() - self.volume = None - self.tempdir = tempfile.mkdtemp() + self.compute = compute_service.ComputeService() self.flags(connection_type='fake', - fake_storage=True, - aoe_export_dir=self.tempdir) + fake_storage=True) self.volume = volume_service.VolumeService() - def tearDown(self): - shutil.rmtree(self.tempdir) - @defer.inlineCallbacks def test_run_create_volume(self): vol_size = '0' user_id = 'fake' project_id = 'fake' volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) - # TODO(termie): get_volume returns differently than create_volume self.assertEqual(volume_id, - volume_service.get_volume(volume_id)['volume_id']) + models.Volume.find(volume_id).id) - rv = self.volume.delete_volume(volume_id) - self.assertRaises(exception.Error, volume_service.get_volume, volume_id) + yield self.volume.delete_volume(volume_id) + self.assertRaises(exception.NotFound, models.Volume.find, volume_id) @defer.inlineCallbacks def test_too_big_volume(self): @@ -100,32 +92,31 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' mountpoint = "/dev/sdf" volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) - volume_obj = volume_service.get_volume(volume_id) - volume_obj.start_attach(instance_id, mountpoint) + vol = models.Volume.find(volume_id) + self.volume.start_attach(volume_id, instance_id, mountpoint) if FLAGS.fake_tests: - volume_obj.finish_attach() + self.volume.finish_attach(volume_id) else: rv = yield self.compute.attach_volume(instance_id, volume_id, mountpoint) - self.assertEqual(volume_obj['status'], "in-use") - self.assertEqual(volume_obj['attach_status'], "attached") - self.assertEqual(volume_obj['instance_id'], instance_id) - self.assertEqual(volume_obj['mountpoint'], mountpoint) + self.assertEqual(vol.status, "in-use") + self.assertEqual(vol.attach_status, "attached") + self.assertEqual(vol.instance_id, instance_id) + self.assertEqual(vol.mountpoint, mountpoint) self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) - volume_obj.start_detach() + self.volume.start_detach(volume_id) if FLAGS.fake_tests: - volume_obj.finish_detach() + self.volume.finish_detach(volume_id) else: rv = yield self.volume.detach_volume(instance_id, volume_id) - volume_obj = volume_service.get_volume(volume_id) - self.assertEqual(volume_obj['status'], "available") + self.assertEqual(vol.status, "available") rv = self.volume.delete_volume(volume_id) self.assertRaises(exception.Error, - volume_service.get_volume, + models.Volume.find, volume_id) @defer.inlineCallbacks @@ -135,7 +126,7 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' shelf_blades = [] def _check(volume_id): - vol = volume_service.get_volume(volume_id) + vol = models.Volume.find(volume_id) shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id']) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) From d8a57343d0262d52fc6baef21b163723d7600ee0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 15 Aug 2010 16:37:06 -0700 Subject: [PATCH 011/113] typos --- nova/tests/volume_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index e979995f..91706580 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -127,11 +127,11 @@ class VolumeTestCase(test.TrialTestCase): shelf_blades = [] def _check(volume_id): vol = models.Volume.find(volume_id) - shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id']) + shelf_blade = '%s.%s' % (vol.shelf_id, vol.blade_id) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) - vol.destroy() + vol.delete() deferreds = [] for i in range(5): d = self.volume.create_volume(vol_size, user_id, project_id) From c2c0f624f628cab7f9104b61c8173354d7e437b3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 16 Aug 2010 01:51:28 -0700 Subject: [PATCH 012/113] fix launching and describing instances to work with sqlalchemy --- nova/endpoint/cloud.py | 106 ++++++++++++++++++++--------------------- nova/models.py | 18 ++++++- 2 files changed, 69 insertions(+), 55 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 5366acec..b68c1345 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -31,6 +31,7 @@ from twisted.internet import defer from nova import datastore from nova import exception from nova import flags +from nova import models from nova import rpc from nova import utils from nova.auth import rbac @@ -403,46 +404,43 @@ class CloudController(object): def _format_instances(self, context, reservation_id = None): reservations = {} if context.user.is_admin(): - instgenerator = self.instdir.all + instgenerator = models.Instance.all() else: - instgenerator = self.instdir.by_project(context.project.id) + instgenerator = models.Instance.all() # FIXME for instance in instgenerator: - res_id = instance.get('reservation_id', 'Unknown') + res_id = instance.reservation_id if reservation_id != None and reservation_id != res_id: continue if not context.user.is_admin(): if instance['image_id'] == FLAGS.vpn_image_id: continue i = {} - i['instance_id'] = instance.get('instance_id', None) - i['image_id'] = instance.get('image_id', None) - i['instance_state'] = { - 'code': instance.get('state', 0), - 'name': instance.get('state_description', 'pending') + i['instanceId'] = instance.name + i['imageId'] = instance.image_id + i['instanceState'] = { + 'code': instance.state, + 'name': instance.state_description } - i['public_dns_name'] = network_model.get_public_ip_for_instance( - i['instance_id']) - i['private_dns_name'] = instance.get('private_dns_name', None) + i['public_dns_name'] = None #network_model.get_public_ip_for_instance( + # i['instance_id']) + i['private_dns_name'] = instance.fixed_ip if not i['public_dns_name']: i['public_dns_name'] = i['private_dns_name'] - i['dns_name'] = instance.get('dns_name', None) - i['key_name'] = instance.get('key_name', None) + i['dns_name'] = None + i['key_name'] = instance.key_name if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), - instance.get('node_name', '')) - i['product_codes_set'] = self._convert_to_set( - instance.get('product_codes', None), 'product_code') - i['instance_type'] = instance.get('instance_type', None) - i['launch_time'] = instance.get('launch_time', None) - i['ami_launch_index'] = instance.get('ami_launch_index', - None) + instance.project_id, + 'node_name') # FIXME + i['product_codes_set'] = self._convert_to_set([], 'product_codes') + i['instance_type'] = instance.instance_type + i['launch_time'] = instance.created_at + i['ami_launch_index'] = instance.launch_index if not reservations.has_key(res_id): r = {} r['reservation_id'] = res_id - r['owner_id'] = instance.get('project_id', None) - r['group_set'] = self._convert_to_set( - instance.get('groups', None), 'group_id') + r['owner_id'] = instance.project_id + r['group_set'] = self._convert_to_set([], 'groups') r['instances_set'] = [] reservations[res_id] = r reservations[res_id]['instances_set'].append(i) @@ -528,7 +526,7 @@ class CloudController(object): defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks + #@defer.inlineCallbacks def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists @@ -560,46 +558,46 @@ class CloudController(object): raise exception.ApiError('Key Pair %s not found' % kwargs['key_name']) key_data = key_pair.public_key - network_topic = yield self._get_network_topic(context) + # network_topic = yield self._get_network_topic(context) # TODO: Get the real security group of launch in here security_group = "default" for num in range(int(kwargs['max_count'])): is_vpn = False if image_id == FLAGS.vpn_image_id: is_vpn = True - inst = self.instdir.new() - allocate_data = yield rpc.call(network_topic, - {"method": "allocate_fixed_ip", - "args": {"user_id": context.user.id, - "project_id": context.project.id, - "security_group": security_group, - "is_vpn": is_vpn, - "hostname": inst.instance_id}}) - inst['image_id'] = image_id - inst['kernel_id'] = kernel_id - inst['ramdisk_id'] = ramdisk_id - inst['user_data'] = kwargs.get('user_data', '') - inst['instance_type'] = kwargs.get('instance_type', 'm1.small') - inst['reservation_id'] = reservation_id - inst['launch_time'] = launch_time - inst['key_data'] = key_data or '' - inst['key_name'] = kwargs.get('key_name', '') - inst['user_id'] = context.user.id - inst['project_id'] = context.project.id - inst['ami_launch_index'] = num - inst['security_group'] = security_group - inst['hostname'] = inst.instance_id + inst = models.Instance() + #allocate_data = yield rpc.call(network_topic, + # {"method": "allocate_fixed_ip", + # "args": {"user_id": context.user.id, + # "project_id": context.project.id, + # "security_group": security_group, + # "is_vpn": is_vpn, + # "hostname": inst.instance_id}}) + allocate_data = {'mac_address': utils.generate_mac(), + 'fixed_ip': '192.168.0.100'} + inst.image_id = image_id + inst.kernel_id = kernel_id + inst.ramdisk_id = ramdisk_id + inst.user_data = kwargs.get('user_data', '') + inst.instance_type = kwargs.get('instance_type', 'm1.small') + inst.reservation_id = reservation_id + inst.key_data = key_data + inst.key_name = kwargs.get('key_name', None) + inst.user_id = context.user.id + inst.project_id = context.project.id + inst.launch_index = num + inst.security_group = security_group + inst.hostname = inst.id for (key, value) in allocate_data.iteritems(): - inst[key] = value - + setattr(inst, key, value) inst.save() rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id": inst.instance_id}}) + "args": {"instance_id": inst.id}}) logging.debug("Casting to node for %s's instance with IP of %s" % - (context.user.name, inst['private_dns_name'])) - # TODO: Make Network figure out the network name from ip. - defer.returnValue(self._format_instances(context, reservation_id)) + (context.user.name, inst.fixed_ip)) + # defer.returnValue(self._format_instances(context, reservation_id)) + return self._format_instances(context, reservation_id) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks diff --git a/nova/models.py b/nova/models.py index c397270d..9cbebca7 100644 --- a/nova/models.py +++ b/nova/models.py @@ -17,7 +17,7 @@ class NovaBase(object): if NovaBase._engine is not None: return NovaBase._engine from sqlalchemy import create_engine - NovaBase._engine = create_engine('sqlite:///:memory:', echo=False) + NovaBase._engine = create_engine('sqlite:////root/nova.sqlite', echo=False) Base.metadata.create_all(NovaBase._engine) return NovaBase._engine @@ -91,6 +91,11 @@ class Network(Base): bridge = Column(String) vlan = Column(String) kind = Column(String) + + @property + def bridge_name(self): + # HACK: this should be set on creation + return 'br100' #vpn_port = Column(Integer) project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) @@ -113,6 +118,12 @@ class Instance(Base, NovaBase): def project(self): return auth.manager.AuthManager().get_project(self.project_id) + # FIXME: make this opaque somehow + @property + def name(self): + return "i-%s" % self.id + + image_id = Column(Integer, ForeignKey('images.id'), nullable=False) kernel_id = Column(String, ForeignKey('images.id'), nullable=True) ramdisk_id = Column(String, ForeignKey('images.id'), nullable=True) @@ -132,12 +143,17 @@ class Instance(Base, NovaBase): user_data = Column(Text) + reservation_id = Column(String) + mac_address = Column(String) + fixed_ip = Column(String) + def set_state(self, state_code, state_description=None): from nova.compute import power_state self.state = state_code if not state_description: state_description = power_state.name(state_code) self.state_description = state_description + self.save() # ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) # kernel = relationship(Kernel, backref=backref('instances', order_by=id)) From f489dc055da93a623f28d89a2d03a9f50ab4baa7 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 17 Aug 2010 16:46:19 +0200 Subject: [PATCH 013/113] Use the argument handler specified by twistd, if any. --- nova/flags.py | 3 +++ nova/server.py | 6 +++++- nova/twistd.py | 12 +++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index e3feb252..e0181102 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -141,6 +141,7 @@ def _wrapper(func): return _wrapped +DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) DEFINE_bool = _wrapper(gflags.DEFINE_bool) @@ -152,6 +153,8 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) +ArgumentSerializer = gflags.ArgumentSerializer + def DECLARE(name, module_string, flag_values=FLAGS): if module_string not in sys.modules: diff --git a/nova/server.py b/nova/server.py index 96550f07..c6b60e09 100644 --- a/nova/server.py +++ b/nova/server.py @@ -44,6 +44,8 @@ flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') flags.DEFINE_string('logfile', None, 'log file to output to') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') +flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') +flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') def stop(pidfile): @@ -135,6 +137,8 @@ def daemonize(args, name, main): threaded=False), stdin=stdin, stdout=stdout, - stderr=stderr + stderr=stderr, + uid=FLAGS.uid, + gid=FLAGS.gid ): main(args) diff --git a/nova/twistd.py b/nova/twistd.py index 8de322aa..a72cc85e 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -48,6 +48,13 @@ class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): return +class FlagParser(object): + def __init__(self, parser): + self.parser = parser + + def Parse(self, s): + return self.parser(s) + def WrapTwistedOptions(wrapped): class TwistedOptionsToFlags(wrapped): @@ -79,7 +86,10 @@ def WrapTwistedOptions(wrapped): reflect.accumulateClassList(self.__class__, 'optParameters', twistd_params) for param in twistd_params: key = param[0].replace('-', '_') - flags.DEFINE_string(key, param[2], str(param[-1])) + if len(param) > 4: + flags.DEFINE(FlagParser(param[4]), key, param[2], str(param[3]), serializer=flags.ArgumentSerializer()) + else: + flags.DEFINE_string(key, param[2], str(param[3])) def _absorbHandlers(self): twistd_handlers = {} From e488bd7edd3c722a0fcba54ba0041f63f6cb4ae0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 13:02:11 -0700 Subject: [PATCH 014/113] start with model code --- nova/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nova/models.py b/nova/models.py index 561a722f..e4075fae 100644 --- a/nova/models.py +++ b/nova/models.py @@ -193,6 +193,18 @@ class Volume(Base, NovaBase): attach_status = Column(String) # FIXME enum delete_on_termination = Column(Boolean) +class Network(Base, NovaBase): + __tablename__ = 'networks' + +class FixedIp(Base, NovaBase): + __tablename__ = 'fixed_ips' + +class ElasticIp(Base, NovaBase): + __tablename__ = 'elastic_ips' + +class Vpn(Base, NovaBase): + __tablename__ = 'vpns' + def create_session(engine=None): return NovaBase.get_session() From 4291ff7e8619ab115a5614bd5047b0feaefe2fa9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 18 Aug 2010 00:05:06 +0200 Subject: [PATCH 015/113] Stylistic improvements. --- nova/flags.py | 2 -- nova/twistd.py | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index e0181102..6f9f906d 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -153,8 +153,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) -ArgumentSerializer = gflags.ArgumentSerializer - def DECLARE(name, module_string, flag_values=FLAGS): if module_string not in sys.modules: diff --git a/nova/twistd.py b/nova/twistd.py index a72cc85e..9511c231 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -21,6 +21,7 @@ Twisted daemon helpers, specifically to parse out gFlags from twisted flags, manage pid files and support syslogging. """ +import gflags import logging import os import signal @@ -48,6 +49,7 @@ class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): return + class FlagParser(object): def __init__(self, parser): self.parser = parser @@ -87,7 +89,9 @@ def WrapTwistedOptions(wrapped): for param in twistd_params: key = param[0].replace('-', '_') if len(param) > 4: - flags.DEFINE(FlagParser(param[4]), key, param[2], str(param[3]), serializer=flags.ArgumentSerializer()) + flags.DEFINE(FlagParser(param[4]), + key, param[2], str(param[3]), + serializer=gflags.ArgumentSerializer()) else: flags.DEFINE_string(key, param[2], str(param[3])) From 683b18ba49d99baabafc15db41e8519709a6e0df Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 16:55:45 -0700 Subject: [PATCH 016/113] network datamodel code --- nova/models.py | 84 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/nova/models.py b/nova/models.py index e4075fae..88627ae0 100644 --- a/nova/models.py +++ b/nova/models.py @@ -89,19 +89,6 @@ class Image(Base, NovaBase): if val != 'machine': assert(val is None) -class Network(Base): - __tablename__ = 'networks' - id = Column(Integer, primary_key=True) - bridge = Column(String) - vlan = Column(String) - kind = Column(String) - - @property - def bridge_name(self): - # HACK: this should be set on creation - return 'br100' - #vpn_port = Column(Integer) - project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) class PhysicalNode(Base): __tablename__ = 'physical_nodes' @@ -186,24 +173,89 @@ class Volume(Base, NovaBase): node_name = Column(String) size = Column(Integer) alvailability_zone = Column(String) # FIXME foreign key? - instance_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) mountpoint = Column(String) attach_time = Column(String) # FIXME datetime status = Column(String) # FIXME enum? attach_status = Column(String) # FIXME enum delete_on_termination = Column(Boolean) + class Network(Base, NovaBase): __tablename__ = 'networks' + id = Column(Integer, primary_key=True) + kind = Column(String) + injected = Column(Boolean) + network_str = Column(String) + netmask = Column(String) + bridge = Column(String) + gateway = Column(String) + broadcast = Column(String) + dns = Column(String) + + vlan = Column(Integer) + vpn_public_ip_str = Column(String) + vpn_public_port = Column(Integer) + vpn_private_ip_str = Column(String) + + project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) + # FIXME: should be physical_node_id = Column(Integer) + node_name = Column(String) + + +class NetworkIndex(Base, NovaBase): + __tablename__ = 'network_indexes' + id = Column(Integer, primary_key=True) + index = Column(Integer) + network_id = Column(Integer, ForeignKey('networks.id'), nullable=True) + network = relationship(Network, backref=backref('vpn', + uselist=False)) + + +#FIXME can these both come from the same baseclass? class FixedIp(Base, NovaBase): __tablename__ = 'fixed_ips' + id = Column(Integer, primary_key=True) + ip_str = Column(String, unique=True) + network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) + network = relationship(Network, backref=backref('fixed_ips')) + instance = relationship(Instance, backref=backref('fixed_ip', + uselist=False)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) + instance = relationship(Instance, backref=backref('fixed_ip', + uselist=False)) + allocated = Column(Boolean) + leased = Column(Boolean) + reserved = Column(Boolean) + + @classmethod + def find_by_ip_str(cls, ip_str): + session = NovaBase.get_session() + try: + return session.query(cls).filter_by(ip_str=ip_str).one() + except exc.NoResultFound: + raise exception.NotFound("No model for ip str %s" % ip_str) class ElasticIp(Base, NovaBase): __tablename__ = 'elastic_ips' + id = Column(Integer, primary_key=True) + ip_str = Column(String, unique=True) + fixed_ip_id = Column(Integer, ForeignKey('fixed_ip.id'), nullable=True) + fixed_ip = relationship(Network, backref=backref('elastic_ips')) + + project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) + # FIXME: should be physical_node_id = Column(Integer) + node_name = Column(String) + + @classmethod + def find_by_ip_str(cls, ip_str): + session = NovaBase.get_session() + try: + return session.query(cls).filter_by(ip_str=ip_str).one() + except exc.NoResultFound: + raise exception.NotFound("No model for ip str %s" % ip_str) -class Vpn(Base, NovaBase): - __tablename__ = 'vpns' def create_session(engine=None): return NovaBase.get_session() From 4e49601a94e5ccc0422e38a80cebb4ae6b64012c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 18:08:39 -0700 Subject: [PATCH 017/113] fix vpn access for auth --- nova/auth/manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index c16eb0c3..d2d4d641 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -34,7 +34,6 @@ from nova import flags from nova import models from nova import utils from nova.auth import signer -from nova.network import vpn FLAGS = flags.FLAGS @@ -571,10 +570,12 @@ class AuthManager(object): @return: A tuple containing (ip, port) or None, None if vpn has not been allocated for user. """ - network_data = vpn.NetworkData.lookup(Project.safe_id(project)) - if not network_data: + # FIXME(vish): this shouldn't be messing with the datamodel directly + if not isinstance(project, Project): + project = self.get_project(project) + if not project.network: raise exception.NotFound('project network data has not been set') - return (network_data.ip, network_data.port) + return (project.network.vpn_ip_str, project.network.vpn_port) def delete_project(self, project): """Deletes a project""" From 478106049746776610cc16e0352e275944a5b654 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 18:10:11 -0700 Subject: [PATCH 018/113] remove references to deleted files so tests run --- nova/endpoint/cloud.py | 1 - nova/tests/network_unittest.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 3bc03e0b..e5d4661d 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -41,7 +41,6 @@ from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES from nova.endpoint import images from nova.network import service as network_service -from nova.network import model as network_model from nova.volume import service diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 03950980..72dc88f2 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -26,9 +26,7 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager -from nova.network import model from nova.network import service -from nova.network import vpn from nova.network.exception import NoMoreAddresses FLAGS = flags.FLAGS From f85228389541dd1cfefe85d10e5f0a8e69867839 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 19:41:17 -0700 Subject: [PATCH 019/113] progress on tests passing --- nova/models.py | 50 +++++++++++++++++++++---- nova/tests/fake_flags.py | 1 + nova/tests/network_unittest.py | 67 +++++++++++++++++++--------------- 3 files changed, 81 insertions(+), 37 deletions(-) diff --git a/nova/models.py b/nova/models.py index 88627ae0..5fc4ba1c 100644 --- a/nova/models.py +++ b/nova/models.py @@ -1,11 +1,43 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +SQLAlchemy models for nova data +""" +import os + from sqlalchemy.orm import relationship, backref, validates, exc -from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime, Boolean, Text +from sqlalchemy import Table, Column, Integer, String +from sqlalchemy import MetaData, ForeignKey, DateTime, Boolean, Text from sqlalchemy.ext.declarative import declarative_base + from nova import auth from nova import exception +from nova import flags + +FLAGS=flags.FLAGS Base = declarative_base() +flags.DEFINE_string('sql_connection', + 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), + 'connection string for sql database') + class NovaBase(object): created_at = Column(DateTime) updated_at = Column(DateTime) @@ -17,7 +49,7 @@ class NovaBase(object): if NovaBase._engine is not None: return NovaBase._engine from sqlalchemy import create_engine - NovaBase._engine = create_engine('sqlite:////root/nova.sqlite', echo=False) + NovaBase._engine = create_engine(FLAGS.sql_connection, echo=False) Base.metadata.create_all(NovaBase._engine) return NovaBase._engine @@ -34,6 +66,11 @@ class NovaBase(object): session = NovaBase.get_session() return session.query(cls).all() + @classmethod + def count(cls): + session = NovaBase.get_session() + return session.query(cls).count() + @classmethod def find(cls, obj_id): session = NovaBase.get_session() @@ -136,7 +173,6 @@ class Instance(Base, NovaBase): reservation_id = Column(String) mac_address = Column(String) - fixed_ip = Column(String) def set_state(self, state_code, state_description=None): from nova.compute import power_state @@ -209,7 +245,7 @@ class NetworkIndex(Base, NovaBase): id = Column(Integer, primary_key=True) index = Column(Integer) network_id = Column(Integer, ForeignKey('networks.id'), nullable=True) - network = relationship(Network, backref=backref('vpn', + network = relationship(Network, backref=backref('network_index', uselist=False)) @@ -220,8 +256,6 @@ class FixedIp(Base, NovaBase): ip_str = Column(String, unique=True) network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) network = relationship(Network, backref=backref('fixed_ips')) - instance = relationship(Instance, backref=backref('fixed_ip', - uselist=False)) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) instance = relationship(Instance, backref=backref('fixed_ip', uselist=False)) @@ -241,8 +275,8 @@ class ElasticIp(Base, NovaBase): __tablename__ = 'elastic_ips' id = Column(Integer, primary_key=True) ip_str = Column(String, unique=True) - fixed_ip_id = Column(Integer, ForeignKey('fixed_ip.id'), nullable=True) - fixed_ip = relationship(Network, backref=backref('elastic_ips')) + fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) + fixed_ip = relationship(FixedIp, backref=backref('elastic_ips')) project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) # FIXME: should be physical_node_id = Column(Integer) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index a7310fb2..ecbc6593 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -26,3 +26,4 @@ FLAGS.fake_rabbit = True FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' FLAGS.verbose = True +FLAGS.sql_connection = 'sqlite:///:memory:' diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 72dc88f2..8b7730d8 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -23,6 +23,7 @@ import os import logging from nova import flags +from nova import models from nova import test from nova import utils from nova.auth import manager @@ -47,16 +48,20 @@ class NetworkTestCase(test.TrialTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] - self.projects.append(self.manager.create_project('netuser', - 'netuser', - 'netuser')) + self.service = service.VlanNetworkService() for i in range(0, 6): name = 'project%s' % i self.projects.append(self.manager.create_project(name, 'netuser', name)) - vpn.NetworkData.create(self.projects[i].id) - self.service = service.VlanNetworkService() + # create the necessary network data for the project + self.service.set_network_host(self.projects[i].id) + instance = models.Instance() + instance.mac_address = utils.generate_mac() + instance.hostname = 'fake' + instance.image_id = 'fake' + instance.save() + self.instance = instance def tearDown(self): # pylint: disable=C0103 super(NetworkTestCase, self).tearDown() @@ -67,32 +72,34 @@ class NetworkTestCase(test.TrialTestCase): def test_public_network_allocation(self): """Makes sure that we can allocaate a public ip""" pubnet = IPy.IP(flags.FLAGS.public_range) - address = self.service.allocate_elastic_ip(self.user.id, - self.projects[0].id) + address = self.service.allocate_elastic_ip(self.projects[0].id) self.assertTrue(IPy.IP(address) in pubnet) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" - result = self.service.allocate_fixed_ip( - self.user.id, self.projects[0].id) - address = result['private_dns_name'] - mac = result['mac_address'] - net = model.get_project_network(self.projects[0].id, "default") + address = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance.id) + net = service.get_project_network(self.projects[0].id) self.assertEqual(True, is_in_project(address, self.projects[0].id)) - hostname = "test-host" - issue_ip(mac, address, hostname, net.bridge_name) + issue_ip(self.instance.mac_address, + address, + self.instance.hostname, + net.bridge) self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released self.assertEqual(True, is_in_project(address, self.projects[0].id)) - release_ip(mac, address, hostname, net.bridge_name) + release_ip(self.instance.mac_address, + address, + self.instance.hostname, + net.bridge) self.assertEqual(False, is_in_project(address, self.projects[0].id)) def test_side_effects(self): """Ensures allocating and releasing has no side effects""" hostname = "side-effect-host" - result = self.service.allocate_fixed_ip(self.user.id, + result = self.service.allocate_fixed_ip( self.projects[0].id) mac = result['mac_address'] address = result['private_dns_name'] @@ -101,8 +108,8 @@ class NetworkTestCase(test.TrialTestCase): secondmac = result['mac_address'] secondaddress = result['private_dns_name'] - net = model.get_project_network(self.projects[0].id, "default") - secondnet = model.get_project_network(self.projects[1].id, "default") + net = service.get_project_network(self.projects[0].id) + secondnet = service.get_project_network(self.projects[1].id) self.assertEqual(True, is_in_project(address, self.projects[0].id)) self.assertEqual(True, is_in_project(secondaddress, @@ -128,7 +135,7 @@ class NetworkTestCase(test.TrialTestCase): def test_subnet_edge(self): """Makes sure that private ips don't overlap""" - result = self.service.allocate_fixed_ip(self.user.id, + result = self.service.allocate_fixed_ip( self.projects[0].id) firstaddress = result['private_dns_name'] hostname = "toomany-hosts" @@ -146,7 +153,7 @@ class NetworkTestCase(test.TrialTestCase): self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] - net = model.get_project_network(project_id, "default") + net = service.get_project_network(project_id) issue_ip(mac, address, hostname, net.bridge_name) issue_ip(mac2, address2, hostname, net.bridge_name) issue_ip(mac3, address3, hostname, net.bridge_name) @@ -162,7 +169,7 @@ class NetworkTestCase(test.TrialTestCase): release_ip(mac, address, hostname, net.bridge_name) release_ip(mac2, address2, hostname, net.bridge_name) release_ip(mac3, address3, hostname, net.bridge_name) - net = model.get_project_network(self.projects[0].id, "default") + net = service.get_project_network(self.projects[0].id) self.service.deallocate_fixed_ip(firstaddress) release_ip(mac, firstaddress, hostname, net.bridge_name) @@ -184,12 +191,12 @@ class NetworkTestCase(test.TrialTestCase): def test_ips_are_reused(self): """Makes sure that ip addresses that are deallocated get reused""" result = self.service.allocate_fixed_ip( - self.user.id, self.projects[0].id) + self.projects[0].id) mac = result['mac_address'] address = result['private_dns_name'] hostname = "reuse-host" - net = model.get_project_network(self.projects[0].id, "default") + net = service.get_project_network(self.projects[0].id) issue_ip(mac, address, hostname, net.bridge_name) self.service.deallocate_fixed_ip(address) @@ -215,7 +222,7 @@ class NetworkTestCase(test.TrialTestCase): There are ips reserved at the bottom and top of the range. services (network, gateway, CloudPipe, broadcast) """ - net = model.get_project_network(self.projects[0].id, "default") + net = service.get_project_network(self.projects[0].id) num_preallocated_ips = len(net.assigned) net_size = flags.FLAGS.network_size num_available_ips = net_size - (net.num_bottom_reserved_ips + @@ -226,7 +233,7 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_addresses(self): """Test for a NoMoreAddresses exception when all fixed ips are used. """ - net = model.get_project_network(self.projects[0].id, "default") + net = service.get_project_network(self.projects[0].id) hostname = "toomany-hosts" macs = {} @@ -234,15 +241,17 @@ class NetworkTestCase(test.TrialTestCase): # Number of availaible ips is len of the available list num_available_ips = len(list(net.available)) for i in range(num_available_ips): - result = self.service.allocate_fixed_ip(self.user.id, + result = self.service.allocate_fixed_ip( self.projects[0].id) macs[i] = result['mac_address'] addresses[i] = result['private_dns_name'] issue_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertEqual(len(list(net.available)), 0) - self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, - self.user.id, self.projects[0].id) + self.assertRaises(NoMoreAddresses, + self.service.allocate_fixed_ip, + self.projects[0].id, + 0) for i in range(len(addresses)): self.service.deallocate_fixed_ip(addresses[i]) @@ -252,7 +261,7 @@ class NetworkTestCase(test.TrialTestCase): def is_in_project(address, project_id): """Returns true if address is in specified project""" - return address in model.get_project_network(project_id).assigned + return models.FixedIp.find_by_ip_str(address) == service.get_project_network(project_id) def binpath(script): From f1ba62e867847757e6462cf2999b165889a4dee7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 17 Aug 2010 20:33:37 -0700 Subject: [PATCH 020/113] almost there --- bin/nova-dhcpbridge | 17 ++++---- nova/models.py | 9 ++--- nova/tests/network_unittest.py | 71 ++++++++++++++++------------------ 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index f70a4482..59381159 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -34,7 +34,6 @@ from nova import flags from nova import rpc from nova import utils from nova.network import linux_net -from nova.network import model from nova.network import service FLAGS = flags.FLAGS @@ -43,11 +42,12 @@ FLAGS = flags.FLAGS def add_lease(_mac, ip, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: + logging.debug("leasing_ip") service.VlanNetworkService().lease_ip(ip) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "lease_ip", - "args": {"fixed_ip": ip}}) + "args": {"fixed_ip_str": ip}}) def old_lease(_mac, _ip, _hostname, _interface): @@ -58,20 +58,18 @@ def old_lease(_mac, _ip, _hostname, _interface): def del_lease(_mac, ip, _hostname, _interface): """Called when a lease expires.""" if FLAGS.fake_rabbit: + logging.debug("releasing_ip") service.VlanNetworkService().release_ip(ip) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "release_ip", - "args": {"fixed_ip": ip}}) + "args": {"fixed_ip_str": ip}}) def init_leases(interface): """Get the list of hosts for an interface.""" - net = model.get_network_by_interface(interface) - res = "" - for address in net.assigned_objs: - res += "%s\n" % linux_net.host_dhcp(address) - return res + network = service.get_network_by_interface(interface) + return linux_net.get_dhcp_hosts(network) def main(): @@ -80,6 +78,9 @@ def main(): utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') + LOG_FILENAME = 'example.log' + logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG) + logging.debug("this is a test") if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True FLAGS.redis_db = 8 diff --git a/nova/models.py b/nova/models.py index 5fc4ba1c..110a4fc8 100644 --- a/nova/models.py +++ b/nova/models.py @@ -214,7 +214,6 @@ class Volume(Base, NovaBase): attach_time = Column(String) # FIXME datetime status = Column(String) # FIXME enum? attach_status = Column(String) # FIXME enum - delete_on_termination = Column(Boolean) class Network(Base, NovaBase): @@ -222,7 +221,7 @@ class Network(Base, NovaBase): id = Column(Integer, primary_key=True) kind = Column(String) - injected = Column(Boolean) + injected = Column(Boolean, default=False) network_str = Column(String) netmask = Column(String) bridge = Column(String) @@ -259,9 +258,9 @@ class FixedIp(Base, NovaBase): instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) instance = relationship(Instance, backref=backref('fixed_ip', uselist=False)) - allocated = Column(Boolean) - leased = Column(Boolean) - reserved = Column(Boolean) + allocated = Column(Boolean, default=False) + leased = Column(Boolean, default=False) + reserved = Column(Boolean, default=False) @classmethod def find_by_ip_str(cls, ip_str): diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 8b7730d8..657dd89d 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -43,7 +43,8 @@ class NetworkTestCase(test.TrialTestCase): fake_storage=True, fake_network=True, auth_driver='nova.auth.ldapdriver.FakeLdapDriver', - network_size=32) + network_size=32, + num_networks=10) logging.getLogger().setLevel(logging.DEBUG) self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') @@ -79,22 +80,16 @@ class NetworkTestCase(test.TrialTestCase): """Makes sure that we can allocate and deallocate a fixed ip""" address = self.service.allocate_fixed_ip(self.projects[0].id, self.instance.id) - net = service.get_project_network(self.projects[0].id) - self.assertEqual(True, is_in_project(address, self.projects[0].id)) - issue_ip(self.instance.mac_address, - address, - self.instance.hostname, - net.bridge) + net = service.get_network_for_project(self.projects[0].id) + self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) + issue_ip(address, net.bridge) self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released - self.assertEqual(True, is_in_project(address, self.projects[0].id)) + self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) - release_ip(self.instance.mac_address, - address, - self.instance.hostname, - net.bridge) - self.assertEqual(False, is_in_project(address, self.projects[0].id)) + release_ip(address, net.bridge) + self.assertEqual(False, is_allocated_in_project(address, self.projects[0].id)) def test_side_effects(self): """Ensures allocating and releasing has no side effects""" @@ -108,13 +103,13 @@ class NetworkTestCase(test.TrialTestCase): secondmac = result['mac_address'] secondaddress = result['private_dns_name'] - net = service.get_project_network(self.projects[0].id) - secondnet = service.get_project_network(self.projects[1].id) + net = service.get_network_for_project(self.projects[0].id) + secondnet = service.get_network_for_project(self.projects[1].id) - self.assertEqual(True, is_in_project(address, self.projects[0].id)) - self.assertEqual(True, is_in_project(secondaddress, + self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) + self.assertEqual(True, is_allocated_in_project(secondaddress, self.projects[1].id)) - self.assertEqual(False, is_in_project(address, self.projects[1].id)) + self.assertEqual(False, is_allocated_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued issue_ip(mac, address, hostname, net.bridge_name) @@ -122,15 +117,15 @@ class NetworkTestCase(test.TrialTestCase): self.service.deallocate_fixed_ip(address) release_ip(mac, address, hostname, net.bridge_name) - self.assertEqual(False, is_in_project(address, self.projects[0].id)) + self.assertEqual(False, is_allocated_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second - self.assertEqual(True, is_in_project(secondaddress, + self.assertEqual(True, is_allocated_in_project(secondaddress, self.projects[1].id)) self.service.deallocate_fixed_ip(secondaddress) release_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) - self.assertEqual(False, is_in_project(secondaddress, + self.assertEqual(False, is_allocated_in_project(secondaddress, self.projects[1].id)) def test_subnet_edge(self): @@ -153,15 +148,15 @@ class NetworkTestCase(test.TrialTestCase): self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] - net = service.get_project_network(project_id) + net = service.get_network_for_project(project_id) issue_ip(mac, address, hostname, net.bridge_name) issue_ip(mac2, address2, hostname, net.bridge_name) issue_ip(mac3, address3, hostname, net.bridge_name) - self.assertEqual(False, is_in_project(address, + self.assertEqual(False, is_allocated_in_project(address, self.projects[0].id)) - self.assertEqual(False, is_in_project(address2, + self.assertEqual(False, is_allocated_in_project(address2, self.projects[0].id)) - self.assertEqual(False, is_in_project(address3, + self.assertEqual(False, is_allocated_in_project(address3, self.projects[0].id)) self.service.deallocate_fixed_ip(address) self.service.deallocate_fixed_ip(address2) @@ -169,7 +164,7 @@ class NetworkTestCase(test.TrialTestCase): release_ip(mac, address, hostname, net.bridge_name) release_ip(mac2, address2, hostname, net.bridge_name) release_ip(mac3, address3, hostname, net.bridge_name) - net = service.get_project_network(self.projects[0].id) + net = service.get_network_for_project(self.projects[0].id) self.service.deallocate_fixed_ip(firstaddress) release_ip(mac, firstaddress, hostname, net.bridge_name) @@ -196,7 +191,7 @@ class NetworkTestCase(test.TrialTestCase): address = result['private_dns_name'] hostname = "reuse-host" - net = service.get_project_network(self.projects[0].id) + net = service.get_network_for_project(self.projects[0].id) issue_ip(mac, address, hostname, net.bridge_name) self.service.deallocate_fixed_ip(address) @@ -222,7 +217,7 @@ class NetworkTestCase(test.TrialTestCase): There are ips reserved at the bottom and top of the range. services (network, gateway, CloudPipe, broadcast) """ - net = service.get_project_network(self.projects[0].id) + net = service.get_network_for_project(self.projects[0].id) num_preallocated_ips = len(net.assigned) net_size = flags.FLAGS.network_size num_available_ips = net_size - (net.num_bottom_reserved_ips + @@ -233,7 +228,7 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_addresses(self): """Test for a NoMoreAddresses exception when all fixed ips are used. """ - net = service.get_project_network(self.projects[0].id) + net = service.get_network_for_project(self.projects[0].id) hostname = "toomany-hosts" macs = {} @@ -259,9 +254,13 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(len(list(net.available)), num_available_ips) -def is_in_project(address, project_id): +def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" - return models.FixedIp.find_by_ip_str(address) == service.get_project_network(project_id) + fixed_ip = models.FixedIp.find_by_ip_str(address) + project_net = service.get_network_for_project(project_id) + print fixed_ip.instance + # instance exists until release + return fixed_ip.instance and project_net == fixed_ip.network def binpath(script): @@ -269,10 +268,9 @@ def binpath(script): return os.path.abspath(os.path.join(__file__, "../../../bin", script)) -def issue_ip(mac, private_ip, hostname, interface): +def issue_ip(private_ip, interface): """Run add command on dhcpbridge""" - cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), - mac, private_ip, hostname) + cmd = "%s add %s fake fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} @@ -280,10 +278,9 @@ def issue_ip(mac, private_ip, hostname, interface): logging.debug("ISSUE_IP: %s, %s ", out, err) -def release_ip(mac, private_ip, hostname, interface): +def release_ip(private_ip, interface): """Run del command on dhcpbridge""" - cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), - mac, private_ip, hostname) + cmd = "%s del %s fake fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} From 8c2e381cf4f678434e5517bc1f56f840dc4d14a2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 02:07:04 -0700 Subject: [PATCH 021/113] network tests pass --- bin/nova-dhcpbridge | 6 +- nova/auth/manager.py | 5 +- nova/tests/network_unittest.py | 259 ++++++++++++++++++--------------- 3 files changed, 146 insertions(+), 124 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 59381159..266fd70c 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -81,13 +81,17 @@ def main(): LOG_FILENAME = 'example.log' logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG) logging.debug("this is a test") + sqlfile = os.environ.get('SQL_DB', '') if int(os.environ.get('TESTING', '0')): + logging.debug("fake rabbit is true") FLAGS.fake_rabbit = True FLAGS.redis_db = 8 - FLAGS.network_size = 32 + FLAGS.network_size = 16 FLAGS.connection_type = 'fake' FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' + FLAGS.num_networks = 5 + FLAGS.sql_connection = 'sqlite:///%s' % sqlfile action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index d2d4d641..69816882 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -573,9 +573,10 @@ class AuthManager(object): # FIXME(vish): this shouldn't be messing with the datamodel directly if not isinstance(project, Project): project = self.get_project(project) - if not project.network: + if not project.network.vpn_public_port: raise exception.NotFound('project network data has not been set') - return (project.network.vpn_ip_str, project.network.vpn_port) + return (project.network.vpn_public_ip_str, + project.network.vpn_public_port) def delete_project(self, project): """Deletes a project""" diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 657dd89d..00aaac34 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -21,6 +21,7 @@ Unit Tests for network code import IPy import os import logging +import tempfile from nova import flags from nova import models @@ -28,7 +29,7 @@ from nova import test from nova import utils from nova.auth import manager from nova.network import service -from nova.network.exception import NoMoreAddresses +from nova.network.exception import NoMoreAddresses, NoMoreNetworks FLAGS = flags.FLAGS @@ -39,18 +40,21 @@ class NetworkTestCase(test.TrialTestCase): super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge + fd, sqlfile = tempfile.mkstemp() + self.sqlfile = os.path.abspath(sqlfile) self.flags(connection_type='fake', + sql_connection='sqlite:///%s' % self.sqlfile, fake_storage=True, fake_network=True, auth_driver='nova.auth.ldapdriver.FakeLdapDriver', - network_size=32, - num_networks=10) + network_size=16, + num_networks=5) logging.getLogger().setLevel(logging.DEBUG) self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.service = service.VlanNetworkService() - for i in range(0, 6): + for i in range(5): name = 'project%s' % i self.projects.append(self.manager.create_project(name, 'netuser', @@ -62,149 +66,145 @@ class NetworkTestCase(test.TrialTestCase): instance.hostname = 'fake' instance.image_id = 'fake' instance.save() - self.instance = instance + self.instance_id = instance.id def tearDown(self): # pylint: disable=C0103 super(NetworkTestCase, self).tearDown() for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) + os.unlink(self.sqlfile) - def test_public_network_allocation(self): + def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" + # FIXME better way of adding elastic ips pubnet = IPy.IP(flags.FLAGS.public_range) - address = self.service.allocate_elastic_ip(self.projects[0].id) - self.assertTrue(IPy.IP(address) in pubnet) + elastic_ip = models.ElasticIp() + elastic_ip.ip_str = str(pubnet[0]) + elastic_ip.node_name = FLAGS.node_name + elastic_ip.save() + eaddress = self.service.allocate_elastic_ip(self.projects[0].id) + faddress = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance_id) + self.assertEqual(eaddress, str(pubnet[0])) + self.service.associate_elastic_ip(eaddress, faddress) + # FIXME datamodel abstraction + self.assertEqual(elastic_ip.fixed_ip.ip_str, faddress) + self.service.disassociate_elastic_ip(eaddress) + self.assertEqual(elastic_ip.fixed_ip, None) + self.service.deallocate_elastic_ip(eaddress) + self.service.deallocate_fixed_ip(faddress) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" address = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance.id) + self.instance_id) net = service.get_network_for_project(self.projects[0].id) - self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) - issue_ip(address, net.bridge) + self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) + issue_ip(address, net.bridge, self.sqlfile) self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released - self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) + self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) - release_ip(address, net.bridge) - self.assertEqual(False, is_allocated_in_project(address, self.projects[0].id)) + release_ip(address, net.bridge, self.sqlfile) + self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) def test_side_effects(self): """Ensures allocating and releasing has no side effects""" - hostname = "side-effect-host" - result = self.service.allocate_fixed_ip( - self.projects[0].id) - mac = result['mac_address'] - address = result['private_dns_name'] - result = self.service.allocate_fixed_ip(self.user, - self.projects[1].id) - secondmac = result['mac_address'] - secondaddress = result['private_dns_name'] + address = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance_id) + address2 = self.service.allocate_fixed_ip(self.projects[1].id, + self.instance_id) net = service.get_network_for_project(self.projects[0].id) - secondnet = service.get_network_for_project(self.projects[1].id) + net2 = service.get_network_for_project(self.projects[1].id) - self.assertEqual(True, is_allocated_in_project(address, self.projects[0].id)) - self.assertEqual(True, is_allocated_in_project(secondaddress, - self.projects[1].id)) - self.assertEqual(False, is_allocated_in_project(address, self.projects[1].id)) + self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) + self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) + self.assertFalse(is_allocated_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued - issue_ip(mac, address, hostname, net.bridge_name) - issue_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) + issue_ip(address, net.bridge, self.sqlfile) + issue_ip(address2, net2.bridge, self.sqlfile) self.service.deallocate_fixed_ip(address) - release_ip(mac, address, hostname, net.bridge_name) - self.assertEqual(False, is_allocated_in_project(address, self.projects[0].id)) + release_ip(address, net.bridge, self.sqlfile) + self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second - self.assertEqual(True, is_allocated_in_project(secondaddress, - self.projects[1].id)) + self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) - self.service.deallocate_fixed_ip(secondaddress) - release_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) - self.assertEqual(False, is_allocated_in_project(secondaddress, - self.projects[1].id)) + self.service.deallocate_fixed_ip(address2) + issue_ip(address2, net.bridge, self.sqlfile) + release_ip(address2, net2.bridge, self.sqlfile) + self.assertFalse(is_allocated_in_project(address2, self.projects[1].id)) def test_subnet_edge(self): """Makes sure that private ips don't overlap""" - result = self.service.allocate_fixed_ip( - self.projects[0].id) - firstaddress = result['private_dns_name'] - hostname = "toomany-hosts" + first = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance_id) for i in range(1, 5): project_id = self.projects[i].id - result = self.service.allocate_fixed_ip( - self.user, project_id) - mac = result['mac_address'] - address = result['private_dns_name'] - result = self.service.allocate_fixed_ip( - self.user, project_id) - mac2 = result['mac_address'] - address2 = result['private_dns_name'] - result = self.service.allocate_fixed_ip( - self.user, project_id) - mac3 = result['mac_address'] - address3 = result['private_dns_name'] + address = self.service.allocate_fixed_ip(project_id, self.instance_id) + address2 = self.service.allocate_fixed_ip(project_id, self.instance_id) + address3 = self.service.allocate_fixed_ip(project_id, self.instance_id) net = service.get_network_for_project(project_id) - issue_ip(mac, address, hostname, net.bridge_name) - issue_ip(mac2, address2, hostname, net.bridge_name) - issue_ip(mac3, address3, hostname, net.bridge_name) - self.assertEqual(False, is_allocated_in_project(address, - self.projects[0].id)) - self.assertEqual(False, is_allocated_in_project(address2, - self.projects[0].id)) - self.assertEqual(False, is_allocated_in_project(address3, - self.projects[0].id)) + issue_ip(address, net.bridge, self.sqlfile) + issue_ip(address2, net.bridge, self.sqlfile) + issue_ip(address3, net.bridge, self.sqlfile) + self.assertFalse(is_allocated_in_project(address, + self.projects[0].id)) + self.assertFalse(is_allocated_in_project(address2, + self.projects[0].id)) + self.assertFalse(is_allocated_in_project(address3, + self.projects[0].id)) self.service.deallocate_fixed_ip(address) self.service.deallocate_fixed_ip(address2) self.service.deallocate_fixed_ip(address3) - release_ip(mac, address, hostname, net.bridge_name) - release_ip(mac2, address2, hostname, net.bridge_name) - release_ip(mac3, address3, hostname, net.bridge_name) + release_ip(address, net.bridge, self.sqlfile) + release_ip(address2, net.bridge, self.sqlfile) + release_ip(address3, net.bridge, self.sqlfile) net = service.get_network_for_project(self.projects[0].id) - self.service.deallocate_fixed_ip(firstaddress) - release_ip(mac, firstaddress, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(first) def test_vpn_ip_and_port_looks_valid(self): """Ensure the vpn ip and port are reasonable""" self.assert_(self.projects[0].vpn_ip) - self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start_port) - self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_end_port) + self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start) + self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_start + + FLAGS.num_networks) - def test_too_many_vpns(self): + def test_too_many_networks(self): """Ensure error is raised if we run out of vpn ports""" - vpns = [] - for i in xrange(vpn.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): - vpns.append(vpn.NetworkData.create("vpnuser%s" % i)) - self.assertRaises(vpn.NoMorePorts, vpn.NetworkData.create, "boom") - for network_datum in vpns: - network_datum.destroy() + projects = [] + networks_left = FLAGS.num_networks - len(self.projects) + for i in range(networks_left): + project = self.manager.create_project('many%s' % i, self.user) + self.service.set_network_host(project.id) + projects.append(project) + project = self.manager.create_project('boom' , self.user) + self.assertRaises(NoMoreNetworks, + self.service.set_network_host, + project.id) + self.manager.delete_project(project) + for project in projects: + self.manager.delete_project(project) + def test_ips_are_reused(self): """Makes sure that ip addresses that are deallocated get reused""" - result = self.service.allocate_fixed_ip( - self.projects[0].id) - mac = result['mac_address'] - address = result['private_dns_name'] - - hostname = "reuse-host" + address = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance_id) net = service.get_network_for_project(self.projects[0].id) - - issue_ip(mac, address, hostname, net.bridge_name) + issue_ip(address, net.bridge, self.sqlfile) self.service.deallocate_fixed_ip(address) - release_ip(mac, address, hostname, net.bridge_name) + release_ip(address, net.bridge, self.sqlfile) - result = self.service.allocate_fixed_ip( - self.user, self.projects[0].id) - secondmac = result['mac_address'] - secondaddress = result['private_dns_name'] - self.assertEqual(address, secondaddress) - issue_ip(secondmac, secondaddress, hostname, net.bridge_name) - self.service.deallocate_fixed_ip(secondaddress) - release_ip(secondmac, secondaddress, hostname, net.bridge_name) + address2 = self.service.allocate_fixed_ip(self.projects[0].id, + self.instance_id) + self.assertEqual(address, address2) + self.service.deallocate_fixed_ip(address2) def test_available_ips(self): """Make sure the number of available ips for the network is correct @@ -217,50 +217,65 @@ class NetworkTestCase(test.TrialTestCase): There are ips reserved at the bottom and top of the range. services (network, gateway, CloudPipe, broadcast) """ - net = service.get_network_for_project(self.projects[0].id) - num_preallocated_ips = len(net.assigned) + network = service.get_network_for_project(self.projects[0].id) net_size = flags.FLAGS.network_size - num_available_ips = net_size - (net.num_bottom_reserved_ips + - num_preallocated_ips + - net.num_top_reserved_ips) - self.assertEqual(num_available_ips, len(list(net.available))) + total_ips = (available_ips(network) + + reserved_ips(network) + + allocated_ips(network)) + self.assertEqual(total_ips, net_size) def test_too_many_addresses(self): """Test for a NoMoreAddresses exception when all fixed ips are used. """ - net = service.get_network_for_project(self.projects[0].id) + network = service.get_network_for_project(self.projects[0].id) - hostname = "toomany-hosts" - macs = {} - addresses = {} # Number of availaible ips is len of the available list - num_available_ips = len(list(net.available)) - for i in range(num_available_ips): - result = self.service.allocate_fixed_ip( - self.projects[0].id) - macs[i] = result['mac_address'] - addresses[i] = result['private_dns_name'] - issue_ip(macs[i], addresses[i], hostname, net.bridge_name) - self.assertEqual(len(list(net.available)), 0) + num_available_ips = available_ips(network) + addresses = [] + for i in range(num_available_ips): + project_id = self.projects[0].id + addresses.append(self.service.allocate_fixed_ip(project_id, + self.instance_id)) + issue_ip(addresses[i],network.bridge, self.sqlfile) + + self.assertEqual(available_ips(network), 0) self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, self.projects[0].id, - 0) + self.instance_id) for i in range(len(addresses)): self.service.deallocate_fixed_ip(addresses[i]) - release_ip(macs[i], addresses[i], hostname, net.bridge_name) - self.assertEqual(len(list(net.available)), num_available_ips) + release_ip(addresses[i],network.bridge, self.sqlfile) + self.assertEqual(available_ips(network), num_available_ips) +# FIXME move these to abstraction layer +def available_ips(network): + session = models.NovaBase.get_session() + query = session.query(models.FixedIp).filter_by(network_id=network.id) + query = query.filter_by(allocated=False).filter_by(reserved=False) + return query.count() + +def allocated_ips(network): + session = models.NovaBase.get_session() + query = session.query(models.FixedIp).filter_by(network_id=network.id) + query = query.filter_by(allocated=True) + return query.count() + +def reserved_ips(network): + session = models.NovaBase.get_session() + query = session.query(models.FixedIp).filter_by(network_id=network.id) + query = query.filter_by(reserved=True) + return query.count() + def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" fixed_ip = models.FixedIp.find_by_ip_str(address) project_net = service.get_network_for_project(project_id) - print fixed_ip.instance # instance exists until release - return fixed_ip.instance and project_net == fixed_ip.network + return fixed_ip.instance is not None and fixed_ip.network == project_net def binpath(script): @@ -268,20 +283,22 @@ def binpath(script): return os.path.abspath(os.path.join(__file__, "../../../bin", script)) -def issue_ip(private_ip, interface): +def issue_ip(private_ip, interface, sqlfile): """Run add command on dhcpbridge""" - cmd = "%s add %s fake fake" % (binpath('nova-dhcpbridge'), private_ip) + cmd = "%s add fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, 'TESTING': '1', + 'SQL_DB': sqlfile, 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s ", out, err) -def release_ip(private_ip, interface): +def release_ip(private_ip, interface, sqlfile): """Run del command on dhcpbridge""" - cmd = "%s del %s fake fake" % (binpath('nova-dhcpbridge'), private_ip) + cmd = "%s del fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, + 'SQL_DB': sqlfile, 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) From 8caba18ea1b6f9233763d56e127dc67de49da700 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 04:08:32 -0700 Subject: [PATCH 022/113] tests pass --- nova/models.py | 10 ++++++++-- nova/tests/volume_unittest.py | 26 +++++++++++++++++++------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/nova/models.py b/nova/models.py index 110a4fc8..6342a86c 100644 --- a/nova/models.py +++ b/nova/models.py @@ -199,8 +199,6 @@ class Volume(Base, NovaBase): __tablename__ = 'volumes' id = Column(Integer, primary_key=True) volume_id = Column(String) - shelf_id = Column(Integer) - blade_id = Column(Integer) user_id = Column(String) #, ForeignKey('users.id'), nullable=False) project_id = Column(String) #, ForeignKey('projects.id')) @@ -215,6 +213,14 @@ class Volume(Base, NovaBase): status = Column(String) # FIXME enum? attach_status = Column(String) # FIXME enum +class ExportDevice(Base, NovaBase): + __tablename__ = 'export_devices' + id = Column(Integer, primary_key=True) + shelf_id = Column(Integer) + blade_id = Column(Integer) + volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume = relationship(Volume, backref=backref('export_device', + uselist=False)) class Network(Base, NovaBase): __tablename__ = 'networks' diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 91706580..f29464ca 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -39,6 +39,20 @@ class VolumeTestCase(test.TrialTestCase): self.flags(connection_type='fake', fake_storage=True) self.volume = volume_service.VolumeService() + self.total_slots = 10 + # FIXME this should test actual creation method + self.devices = [] + for i in xrange(self.total_slots): + export_device = models.ExportDevice() + export_device.shelf_id = 0 + export_device.blade_id = i + export_device.save() + self.devices.append(export_device) + + def tearDown(self): + super(VolumeTestCase, self).tearDown() + for device in self.devices: + device.delete() @defer.inlineCallbacks def test_run_create_volume(self): @@ -68,14 +82,11 @@ class VolumeTestCase(test.TrialTestCase): vol_size = '1' user_id = 'fake' project_id = 'fake' - num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1 - total_slots = FLAGS.blades_per_shelf * num_shelves vols = [] - from nova import datastore - redis = datastore.Redis.instance() - for i in xrange(total_slots): + for i in xrange(self.total_slots): vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) + print models.Volume.find(vid).export_device.volume_id self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), @@ -127,13 +138,14 @@ class VolumeTestCase(test.TrialTestCase): shelf_blades = [] def _check(volume_id): vol = models.Volume.find(volume_id) - shelf_blade = '%s.%s' % (vol.shelf_id, vol.blade_id) + shelf_blade = '%s.%s' % (vol.export_device.shelf_id, + vol.export_device.blade_id) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) vol.delete() deferreds = [] - for i in range(5): + for i in range(self.total_slots): d = self.volume.create_volume(vol_size, user_id, project_id) d.addCallback(_check) d.addErrback(self.fail) From a19a0bf3786778d04f24e888d8393386e934d9bf Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 18 Aug 2010 11:19:40 -0400 Subject: [PATCH 023/113] Image API work --- nova/endpoint/newapi.py | 4 -- nova/endpoint/rackspace/controllers/base.py | 9 ++++ nova/endpoint/rackspace/controllers/images.py | 48 ++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/nova/endpoint/newapi.py b/nova/endpoint/newapi.py index 9aae933a..7836be58 100644 --- a/nova/endpoint/newapi.py +++ b/nova/endpoint/newapi.py @@ -41,11 +41,7 @@ class APIVersionRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - rsapi = rackspace.API() mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) - mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) - super(APIVersionRouter, self).__init__(mapper) - diff --git a/nova/endpoint/rackspace/controllers/base.py b/nova/endpoint/rackspace/controllers/base.py index 8cd44f62..88922280 100644 --- a/nova/endpoint/rackspace/controllers/base.py +++ b/nova/endpoint/rackspace/controllers/base.py @@ -7,3 +7,12 @@ class BaseController(wsgi.Controller): return { cls.entity_name : cls.render(instance) } else: return { "TODO": "TODO" } + + def serialize(self, data, request): + """ + Serialize the given dict to the response type requested in request. + Uses self._serialization_metadata if it exists, which is a dict mapping + MIME types to information needed to serialize to that type. + """ + _metadata = getattr(type(self), "_serialization_metadata", {}) + return Serializer(request.environ, _metadata).to_content_type(data) diff --git a/nova/endpoint/rackspace/controllers/images.py b/nova/endpoint/rackspace/controllers/images.py index ae2a0884..197d8375 100644 --- a/nova/endpoint/rackspace/controllers/images.py +++ b/nova/endpoint/rackspace/controllers/images.py @@ -1 +1,47 @@ -class ImagesController(object): pass +from nova.endpoint.rackspace.controllers.base import BaseController +from nova.endpoint import images +from webob import exc + +#TODO(gundlach): Serialize return values +class ImagesController(BaseController): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "image": [ "id", "name", "updated", "created", "status", + "serverId", "progress" ] + } + } + } + + def index(self, req): + context = req.environ['nova.api_request_context'] + return images.list(context) + + def show(self, req, id): + context = req.environ['nova.api_request_context'] + return images.list(context, filter_list=[id]) + + def delete(self, req, id): + context = req.environ['nova.api_request_context'] + # TODO(gundlach): make sure it's an image they may delete? + return images.deregister(context, id) + + def create(self, **kwargs): + # TODO(gundlach): no idea how to hook this up. code below + # is from servers.py. + inst = self.build_server_instance(kwargs['server']) + rpc.cast( + FLAGS.compute_topic, { + "method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + + def update(self, **kwargs): + # TODO (gundlach): no idea how to hook this up. code below + # is from servers.py. + instance_id = kwargs['id'] + instance = compute.InstanceDirectory().get(instance_id) + if not instance: + raise ServerNotFound("The requested server was not found") + instance.update(kwargs['server']) + instance.save() From ab6e47f1d6316cac52f826bf0cb3cf557ef7f6ec Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 13:11:49 -0700 Subject: [PATCH 024/113] use with_lockmode for concurrency issues --- nova/models.py | 87 ++++++++++++++++++----------------- nova/tests/volume_unittest.py | 8 +++- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/nova/models.py b/nova/models.py index 6342a86c..aa9f3da0 100644 --- a/nova/models.py +++ b/nova/models.py @@ -39,6 +39,7 @@ flags.DEFINE_string('sql_connection', 'connection string for sql database') class NovaBase(object): + __table_args__ = {'mysql_engine':'InnoDB'} created_at = Column(DateTime) updated_at = Column(DateTime) @@ -96,17 +97,17 @@ class NovaBase(object): class Image(Base, NovaBase): __tablename__ = 'images' - user_id = Column(String)#, ForeignKey('users.id'), nullable=False) - project_id = Column(String)#, ForeignKey('projects.id'), nullable=False) + id = Column(Integer, primary_key=True) + user_id = Column(String(255))#, ForeignKey('users.id'), nullable=False) + project_id = Column(String(255))#, ForeignKey('projects.id'), nullable=False) - id = Column(String, primary_key=True) - image_type = Column(String) + image_type = Column(String(255)) public = Column(Boolean, default=False) - state = Column(String) - location = Column(String) - arch = Column(String) - default_kernel_id = Column(String) - default_ramdisk_id = Column(String) + state = Column(String(255)) + location = Column(String(255)) + arch = Column(String(255)) + default_kernel_id = Column(String(255)) + default_ramdisk_id = Column(String(255)) @validates('image_type') def validate_image_type(self, key, image_type): @@ -135,8 +136,8 @@ class Instance(Base, NovaBase): __tablename__ = 'instances' id = Column(Integer, primary_key=True) - user_id = Column(String) #, ForeignKey('users.id'), nullable=False) - project_id = Column(String) #, ForeignKey('projects.id')) + user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) + project_id = Column(String(255)) #, ForeignKey('projects.id')) @property def user(self): @@ -153,26 +154,26 @@ class Instance(Base, NovaBase): image_id = Column(Integer, ForeignKey('images.id'), nullable=False) - kernel_id = Column(String, ForeignKey('images.id'), nullable=True) - ramdisk_id = Column(String, ForeignKey('images.id'), nullable=True) + kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) + ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) launch_index = Column(Integer) - key_name = Column(String) + key_name = Column(String(255)) key_data = Column(Text) - security_group = Column(String) + security_group = Column(String(255)) state = Column(Integer) - state_description = Column(String) + state_description = Column(String(255)) - hostname = Column(String) + hostname = Column(String(255)) physical_node_id = Column(Integer) instance_type = Column(Integer) user_data = Column(Text) - reservation_id = Column(String) - mac_address = Column(String) + reservation_id = Column(String(255)) + mac_address = Column(String(255)) def set_state(self, state_code, state_description=None): from nova.compute import power_state @@ -198,20 +199,20 @@ class Instance(Base, NovaBase): class Volume(Base, NovaBase): __tablename__ = 'volumes' id = Column(Integer, primary_key=True) - volume_id = Column(String) + volume_id = Column(String(255)) - user_id = Column(String) #, ForeignKey('users.id'), nullable=False) - project_id = Column(String) #, ForeignKey('projects.id')) + user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) + project_id = Column(String(255)) #, ForeignKey('projects.id')) # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String) + node_name = Column(String(255)) size = Column(Integer) - alvailability_zone = Column(String) # FIXME foreign key? + alvailability_zone = Column(String(255)) # FIXME foreign key? instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) - mountpoint = Column(String) - attach_time = Column(String) # FIXME datetime - status = Column(String) # FIXME enum? - attach_status = Column(String) # FIXME enum + mountpoint = Column(String(255)) + attach_time = Column(String(255)) # FIXME datetime + status = Column(String(255)) # FIXME enum? + attach_status = Column(String(255)) # FIXME enum class ExportDevice(Base, NovaBase): __tablename__ = 'export_devices' @@ -225,24 +226,24 @@ class ExportDevice(Base, NovaBase): class Network(Base, NovaBase): __tablename__ = 'networks' id = Column(Integer, primary_key=True) - kind = Column(String) + kind = Column(String(255)) injected = Column(Boolean, default=False) - network_str = Column(String) - netmask = Column(String) - bridge = Column(String) - gateway = Column(String) - broadcast = Column(String) - dns = Column(String) + network_str = Column(String(255)) + netmask = Column(String(255)) + bridge = Column(String(255)) + gateway = Column(String(255)) + broadcast = Column(String(255)) + dns = Column(String(255)) vlan = Column(Integer) - vpn_public_ip_str = Column(String) + vpn_public_ip_str = Column(String(255)) vpn_public_port = Column(Integer) - vpn_private_ip_str = Column(String) + vpn_private_ip_str = Column(String(255)) - project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) + project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String) + node_name = Column(String(255)) class NetworkIndex(Base, NovaBase): @@ -258,7 +259,7 @@ class NetworkIndex(Base, NovaBase): class FixedIp(Base, NovaBase): __tablename__ = 'fixed_ips' id = Column(Integer, primary_key=True) - ip_str = Column(String, unique=True) + ip_str = Column(String(255), unique=True) network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) network = relationship(Network, backref=backref('fixed_ips')) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) @@ -279,13 +280,13 @@ class FixedIp(Base, NovaBase): class ElasticIp(Base, NovaBase): __tablename__ = 'elastic_ips' id = Column(Integer, primary_key=True) - ip_str = Column(String, unique=True) + ip_str = Column(String(255), unique=True) fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) fixed_ip = relationship(FixedIp, backref=backref('elastic_ips')) - project_id = Column(String) #, ForeignKey('projects.id'), nullable=False) + project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String) + node_name = Column(String(255)) @classmethod def find_by_ip_str(cls, ip_str): diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index f29464ca..62ea2a26 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -131,19 +131,20 @@ class VolumeTestCase(test.TrialTestCase): volume_id) @defer.inlineCallbacks - def test_multiple_volume_race_condition(self): + def test_concurrent_volumes_get_different_blades(self): vol_size = "5" user_id = "fake" project_id = 'fake' shelf_blades = [] + volume_ids = [] def _check(volume_id): + volume_ids.append(volume_id) vol = models.Volume.find(volume_id) shelf_blade = '%s.%s' % (vol.export_device.shelf_id, vol.export_device.blade_id) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) - vol.delete() deferreds = [] for i in range(self.total_slots): d = self.volume.create_volume(vol_size, user_id, project_id) @@ -151,6 +152,9 @@ class VolumeTestCase(test.TrialTestCase): d.addErrback(self.fail) deferreds.append(d) yield defer.DeferredList(deferreds) + for volume_id in volume_ids: + vol = models.Volume.find(volume_id) + vol.delete() def test_multi_node(self): # TODO(termie): Figure out how to test with two nodes, From 427676cbf1c41e41fbcc7de4c5dc713b8d882436 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 17:38:51 -0700 Subject: [PATCH 025/113] fixing more network issues --- bin/nova-dhcpbridge | 10 +-- nova/auth/manager.py | 20 ++--- nova/models.py | 137 +++++++++++++++++++++------------ nova/tests/auth_unittest.py | 1 - nova/tests/network_unittest.py | 28 ++++--- nova/tests/volume_unittest.py | 1 - run_tests.py | 2 +- 7 files changed, 121 insertions(+), 78 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 266fd70c..bd8fd978 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -35,6 +35,7 @@ from nova import rpc from nova import utils from nova.network import linux_net from nova.network import service +from nova import datastore # for redis_db flag FLAGS = flags.FLAGS @@ -43,6 +44,8 @@ def add_lease(_mac, ip, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: logging.debug("leasing_ip") + print FLAGS.redis_db + print FLAGS.sql_connection service.VlanNetworkService().lease_ip(ip) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), @@ -78,12 +81,8 @@ def main(): utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') - LOG_FILENAME = 'example.log' - logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG) - logging.debug("this is a test") sqlfile = os.environ.get('SQL_DB', '') if int(os.environ.get('TESTING', '0')): - logging.debug("fake rabbit is true") FLAGS.fake_rabbit = True FLAGS.redis_db = 8 FLAGS.network_size = 16 @@ -91,7 +90,8 @@ def main(): FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' FLAGS.num_networks = 5 - FLAGS.sql_connection = 'sqlite:///%s' % sqlfile + FLAGS.sql_connection = 'mysql://root@localhost/test' + #FLAGS.sql_connection = 'sqlite:///%s' % sqlfile action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 69816882..eed67d8c 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -529,11 +529,9 @@ class AuthManager(object): member_users) if project_dict: project = Project(**project_dict) - # FIXME(ja): EVIL HACK - this should poll from a pool - session = models.create_session() - net = models.Network(project_id=project.id, kind='vlan') - session.add(net) - session.commit() + # FIXME(ja): EVIL HACK + net = models.Network(project_id=project.id) + net.save() return project def add_to_project(self, user, project): @@ -580,6 +578,10 @@ class AuthManager(object): def delete_project(self, project): """Deletes a project""" + # FIXME(ja): EVIL HACK + if not isinstance(project, Project): + project = self.get_project(project) + project.network.delete() with self.driver() as drv: return drv.delete_project(Project.safe_id(project)) @@ -714,15 +716,15 @@ class AuthManager(object): zippy.writestr(FLAGS.credential_key_file, private_key) zippy.writestr(FLAGS.credential_cert_file, signed_cert) - network_data = vpn.NetworkData.lookup(pid) - if network_data: + (vpn_ip, vpn_port) = self.get_project_vpn_data(project) + if vpn_ip: configfile = open(FLAGS.vpn_client_template,"r") s = string.Template(configfile.read()) configfile.close() config = s.substitute(keyfile=FLAGS.credential_key_file, certfile=FLAGS.credential_cert_file, - ip=network_data.ip, - port=network_data.port) + ip=vpn_ip, + port=vpn_port) zippy.writestr(FLAGS.credential_vpn_file, config) else: logging.warn("No vpn data for project %s" % diff --git a/nova/models.py b/nova/models.py index aa9f3da0..70010eab 100644 --- a/nova/models.py +++ b/nova/models.py @@ -65,19 +65,24 @@ class NovaBase(object): @classmethod def all(cls): session = NovaBase.get_session() - return session.query(cls).all() + result = session.query(cls).all() + session.commit() + return result @classmethod def count(cls): session = NovaBase.get_session() - return session.query(cls).count() + result = session.query(cls).count() + session.commit() + return result @classmethod def find(cls, obj_id): session = NovaBase.get_session() - #print cls try: - return session.query(cls).filter_by(id=obj_id).one() + result = session.query(cls).filter_by(id=obj_id).one() + session.commit() + return result except exc.NoResultFound: raise exception.NotFound("No model for id %s" % obj_id) @@ -89,12 +94,13 @@ class NovaBase(object): def delete(self): session = NovaBase.get_session() session.delete(self) - session.flush() + session.commit() def refresh(self): session = NovaBase.get_session() session.refresh(self) + class Image(Base, NovaBase): __tablename__ = 'images' id = Column(Integer, primary_key=True) @@ -128,9 +134,29 @@ class Image(Base, NovaBase): assert(val is None) -class PhysicalNode(Base): +class PhysicalNode(Base, NovaBase): __tablename__ = 'physical_nodes' + id = Column(String(255), primary_key=True) + +class Daemon(Base, NovaBase): + __tablename__ = 'daemons' id = Column(Integer, primary_key=True) + node_name = Column(String(255)) #, ForeignKey('physical_node.id')) + binary = Column(String(255)) + report_count = Column(Integer) + + @classmethod + def find_by_args(cls, node_name, binary): + session = NovaBase.get_session() + try: + query = session.query(cls).filter_by(node_name=node_name) + result = query.filter_by(binary=binary).one() + session.commit() + return result + except exc.NoResultFound: + raise exception.NotFound("No model for %s, %s" % (node_name, + binary)) + class Instance(Base, NovaBase): __tablename__ = 'instances' @@ -153,7 +179,7 @@ class Instance(Base, NovaBase): return "i-%s" % self.id - image_id = Column(Integer, ForeignKey('images.id'), nullable=False) + image_id = Column(Integer, ForeignKey('images.id'), nullable=True) kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) @@ -204,8 +230,7 @@ class Volume(Base, NovaBase): user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) project_id = Column(String(255)) #, ForeignKey('projects.id')) - # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String(255)) + node_name = Column(String(255)) #, ForeignKey('physical_node.id')) size = Column(Integer) alvailability_zone = Column(String(255)) # FIXME foreign key? instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) @@ -223,6 +248,52 @@ class ExportDevice(Base, NovaBase): volume = relationship(Volume, backref=backref('export_device', uselist=False)) + +#FIXME can these both come from the same baseclass? +class FixedIp(Base, NovaBase): + __tablename__ = 'fixed_ips' + id = Column(Integer, primary_key=True) + ip_str = Column(String(255), unique=True) + network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) + instance = relationship(Instance, backref=backref('fixed_ip', + uselist=False)) + allocated = Column(Boolean, default=False) + leased = Column(Boolean, default=False) + reserved = Column(Boolean, default=False) + + @classmethod + def find_by_ip_str(cls, ip_str): + session = NovaBase.get_session() + try: + result = session.query(cls).filter_by(ip_str=ip_str).one() + session.commit() + return result + except exc.NoResultFound: + raise exception.NotFound("No model for ip str %s" % ip_str) + + +class ElasticIp(Base, NovaBase): + __tablename__ = 'elastic_ips' + id = Column(Integer, primary_key=True) + ip_str = Column(String(255), unique=True) + fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) + fixed_ip = relationship(FixedIp, backref=backref('elastic_ips')) + + project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) + node_name = Column(String(255)) #, ForeignKey('physical_node.id')) + + @classmethod + def find_by_ip_str(cls, ip_str): + session = NovaBase.get_session() + try: + result = session.query(cls).filter_by(ip_str=ip_str).one() + session.commit() + return result + except exc.NoResultFound: + raise exception.NotFound("No model for ip str %s" % ip_str) + + class Network(Base, NovaBase): __tablename__ = 'networks' id = Column(Integer, primary_key=True) @@ -242,8 +313,12 @@ class Network(Base, NovaBase): vpn_private_ip_str = Column(String(255)) project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) - # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String(255)) + node_name = Column(String(255)) #, ForeignKey('physical_node.id')) + + fixed_ips = relationship(FixedIp, + single_parent=True, + backref=backref('network'), + cascade='all, delete, delete-orphan') class NetworkIndex(Base, NovaBase): @@ -255,46 +330,6 @@ class NetworkIndex(Base, NovaBase): uselist=False)) -#FIXME can these both come from the same baseclass? -class FixedIp(Base, NovaBase): - __tablename__ = 'fixed_ips' - id = Column(Integer, primary_key=True) - ip_str = Column(String(255), unique=True) - network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) - network = relationship(Network, backref=backref('fixed_ips')) - instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) - instance = relationship(Instance, backref=backref('fixed_ip', - uselist=False)) - allocated = Column(Boolean, default=False) - leased = Column(Boolean, default=False) - reserved = Column(Boolean, default=False) - - @classmethod - def find_by_ip_str(cls, ip_str): - session = NovaBase.get_session() - try: - return session.query(cls).filter_by(ip_str=ip_str).one() - except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % ip_str) - -class ElasticIp(Base, NovaBase): - __tablename__ = 'elastic_ips' - id = Column(Integer, primary_key=True) - ip_str = Column(String(255), unique=True) - fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) - fixed_ip = relationship(FixedIp, backref=backref('elastic_ips')) - - project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) - # FIXME: should be physical_node_id = Column(Integer) - node_name = Column(String(255)) - - @classmethod - def find_by_ip_str(cls, ip_str): - session = NovaBase.get_session() - try: - return session.query(cls).filter_by(ip_str=ip_str).one() - except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % ip_str) def create_session(engine=None): diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 0b404bfd..59a81818 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -32,7 +32,6 @@ FLAGS = flags.FLAGS class AuthTestCase(test.BaseTestCase): - flush_db = False def setUp(self): super(AuthTestCase, self).setUp() self.flags(connection_type='fake', diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 00aaac34..c94c81f7 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -23,6 +23,7 @@ import os import logging import tempfile +from nova import exception from nova import flags from nova import models from nova import test @@ -40,10 +41,10 @@ class NetworkTestCase(test.TrialTestCase): super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge - fd, sqlfile = tempfile.mkstemp() - self.sqlfile = os.path.abspath(sqlfile) + self.sqlfile = 'test.sqlite' self.flags(connection_type='fake', - sql_connection='sqlite:///%s' % self.sqlfile, + #sql_connection='sqlite:///%s' % self.sqlfile, + sql_connection='mysql://root@localhost/test', fake_storage=True, fake_network=True, auth_driver='nova.auth.ldapdriver.FakeLdapDriver', @@ -53,6 +54,7 @@ class NetworkTestCase(test.TrialTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] + print FLAGS.sql_connection self.service = service.VlanNetworkService() for i in range(5): name = 'project%s' % i @@ -64,7 +66,6 @@ class NetworkTestCase(test.TrialTestCase): instance = models.Instance() instance.mac_address = utils.generate_mac() instance.hostname = 'fake' - instance.image_id = 'fake' instance.save() self.instance_id = instance.id @@ -73,16 +74,19 @@ class NetworkTestCase(test.TrialTestCase): for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) - os.unlink(self.sqlfile) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" # FIXME better way of adding elastic ips pubnet = IPy.IP(flags.FLAGS.public_range) - elastic_ip = models.ElasticIp() - elastic_ip.ip_str = str(pubnet[0]) - elastic_ip.node_name = FLAGS.node_name - elastic_ip.save() + ip_str = str(pubnet[0]) + try: + elastic_ip = models.ElasticIp.find_by_ip_str(ip_str) + except exception.NotFound: + elastic_ip = models.ElasticIp() + elastic_ip.ip_str = ip_str + elastic_ip.node_name = FLAGS.node_name + elastic_ip.save() eaddress = self.service.allocate_elastic_ip(self.projects[0].id) faddress = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) @@ -101,7 +105,11 @@ class NetworkTestCase(test.TrialTestCase): self.instance_id) net = service.get_network_for_project(self.projects[0].id) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) + print 'I just got allocated' issue_ip(address, net.bridge, self.sqlfile) + obj = models.FixedIp.find_by_ip_str(address) + obj.refresh() + print obj.leased self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released @@ -178,7 +186,7 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_networks(self): """Ensure error is raised if we run out of vpn ports""" projects = [] - networks_left = FLAGS.num_networks - len(self.projects) + networks_left = FLAGS.num_networks - models.Network.count() for i in range(networks_left): project = self.manager.create_project('many%s' % i, self.user) self.service.set_network_host(project.id) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 62ea2a26..82f71901 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -86,7 +86,6 @@ class VolumeTestCase(test.TrialTestCase): for i in xrange(self.total_slots): vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) - print models.Volume.find(vid).export_device.volume_id self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), diff --git a/run_tests.py b/run_tests.py index 77aa9088..82c1aa9c 100644 --- a/run_tests.py +++ b/run_tests.py @@ -55,7 +55,7 @@ from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * -from nova.tests.model_unittest import * +#from nova.tests.model_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * From b6eae44e2e76f486d7a8f2367964c4094d9c47a7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 18:03:58 -0700 Subject: [PATCH 026/113] last few test fixes --- bin/nova-dhcpbridge | 14 +++++++--- nova/tests/fake_flags.py | 3 ++- nova/tests/network_unittest.py | 48 ++++++++++++++-------------------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index bd8fd978..b17a56e6 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -44,7 +44,9 @@ def add_lease(_mac, ip, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: logging.debug("leasing_ip") - print FLAGS.redis_db + from nova import models + print models.FixedIp.count() + print models.Network.count() print FLAGS.sql_connection service.VlanNetworkService().lease_ip(ip) else: @@ -81,7 +83,6 @@ def main(): utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') - sqlfile = os.environ.get('SQL_DB', '') if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True FLAGS.redis_db = 8 @@ -90,8 +91,13 @@ def main(): FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' FLAGS.num_networks = 5 - FLAGS.sql_connection = 'mysql://root@localhost/test' - #FLAGS.sql_connection = 'sqlite:///%s' % sqlfile + path = os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', + '_trial_temp', + 'nova.sqlite')) + print path + FLAGS.sql_connection = 'sqlite:///%s' % path + #FLAGS.sql_connection = 'mysql://root@localhost/test' action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index ecbc6593..7fc83bab 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -26,4 +26,5 @@ FLAGS.fake_rabbit = True FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///:memory:' +FLAGS.sql_connection = 'sqlite:///nova.sqlite' +#FLAGS.sql_connection = 'mysql://root@localhost/test' diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index c94c81f7..0f2ce060 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -41,10 +41,7 @@ class NetworkTestCase(test.TrialTestCase): super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge - self.sqlfile = 'test.sqlite' self.flags(connection_type='fake', - #sql_connection='sqlite:///%s' % self.sqlfile, - sql_connection='mysql://root@localhost/test', fake_storage=True, fake_network=True, auth_driver='nova.auth.ldapdriver.FakeLdapDriver', @@ -54,7 +51,6 @@ class NetworkTestCase(test.TrialTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] - print FLAGS.sql_connection self.service = service.VlanNetworkService() for i in range(5): name = 'project%s' % i @@ -105,17 +101,13 @@ class NetworkTestCase(test.TrialTestCase): self.instance_id) net = service.get_network_for_project(self.projects[0].id) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) - print 'I just got allocated' - issue_ip(address, net.bridge, self.sqlfile) - obj = models.FixedIp.find_by_ip_str(address) - obj.refresh() - print obj.leased + issue_ip(address, net.bridge) self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) - release_ip(address, net.bridge, self.sqlfile) + release_ip(address, net.bridge) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) def test_side_effects(self): @@ -133,19 +125,19 @@ class NetworkTestCase(test.TrialTestCase): self.assertFalse(is_allocated_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued - issue_ip(address, net.bridge, self.sqlfile) - issue_ip(address2, net2.bridge, self.sqlfile) + issue_ip(address, net.bridge) + issue_ip(address2, net2.bridge) self.service.deallocate_fixed_ip(address) - release_ip(address, net.bridge, self.sqlfile) + release_ip(address, net.bridge) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) self.service.deallocate_fixed_ip(address2) - issue_ip(address2, net.bridge, self.sqlfile) - release_ip(address2, net2.bridge, self.sqlfile) + issue_ip(address2, net.bridge) + release_ip(address2, net2.bridge) self.assertFalse(is_allocated_in_project(address2, self.projects[1].id)) def test_subnet_edge(self): @@ -158,9 +150,9 @@ class NetworkTestCase(test.TrialTestCase): address2 = self.service.allocate_fixed_ip(project_id, self.instance_id) address3 = self.service.allocate_fixed_ip(project_id, self.instance_id) net = service.get_network_for_project(project_id) - issue_ip(address, net.bridge, self.sqlfile) - issue_ip(address2, net.bridge, self.sqlfile) - issue_ip(address3, net.bridge, self.sqlfile) + issue_ip(address, net.bridge) + issue_ip(address2, net.bridge) + issue_ip(address3, net.bridge) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) self.assertFalse(is_allocated_in_project(address2, @@ -170,9 +162,9 @@ class NetworkTestCase(test.TrialTestCase): self.service.deallocate_fixed_ip(address) self.service.deallocate_fixed_ip(address2) self.service.deallocate_fixed_ip(address3) - release_ip(address, net.bridge, self.sqlfile) - release_ip(address2, net.bridge, self.sqlfile) - release_ip(address3, net.bridge, self.sqlfile) + release_ip(address, net.bridge) + release_ip(address2, net.bridge) + release_ip(address3, net.bridge) net = service.get_network_for_project(self.projects[0].id) self.service.deallocate_fixed_ip(first) @@ -205,9 +197,9 @@ class NetworkTestCase(test.TrialTestCase): address = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) net = service.get_network_for_project(self.projects[0].id) - issue_ip(address, net.bridge, self.sqlfile) + issue_ip(address, net.bridge) self.service.deallocate_fixed_ip(address) - release_ip(address, net.bridge, self.sqlfile) + release_ip(address, net.bridge) address2 = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) @@ -245,7 +237,7 @@ class NetworkTestCase(test.TrialTestCase): project_id = self.projects[0].id addresses.append(self.service.allocate_fixed_ip(project_id, self.instance_id)) - issue_ip(addresses[i],network.bridge, self.sqlfile) + issue_ip(addresses[i],network.bridge) self.assertEqual(available_ips(network), 0) self.assertRaises(NoMoreAddresses, @@ -255,7 +247,7 @@ class NetworkTestCase(test.TrialTestCase): for i in range(len(addresses)): self.service.deallocate_fixed_ip(addresses[i]) - release_ip(addresses[i],network.bridge, self.sqlfile) + release_ip(addresses[i],network.bridge) self.assertEqual(available_ips(network), num_available_ips) @@ -291,22 +283,20 @@ def binpath(script): return os.path.abspath(os.path.join(__file__, "../../../bin", script)) -def issue_ip(private_ip, interface, sqlfile): +def issue_ip(private_ip, interface): """Run add command on dhcpbridge""" cmd = "%s add fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, 'TESTING': '1', - 'SQL_DB': sqlfile, 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s ", out, err) -def release_ip(private_ip, interface, sqlfile): +def release_ip(private_ip, interface): """Run del command on dhcpbridge""" cmd = "%s del fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) env = {'DNSMASQ_INTERFACE': interface, - 'SQL_DB': sqlfile, 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) From b983ec373f96a178d736c6622c0a51b847ba8a94 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 18 Aug 2010 18:25:16 -0700 Subject: [PATCH 027/113] Removed old cloud_topic queue setup, it is no longer used. --- bin/nova-api | 8 -------- nova/endpoint/cloud.py | 1 - nova/flags.py | 1 - nova/tests/cloud_unittest.py | 4 ---- 4 files changed, 14 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 13baf22a..a3ad5a0e 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -26,7 +26,6 @@ from tornado import httpserver from tornado import ioloop from nova import flags -from nova import rpc from nova import server from nova import utils from nova.endpoint import admin @@ -43,14 +42,7 @@ def main(_argv): 'Admin': admin.AdminController()} _app = api.APIServerApplication(controllers) - conn = rpc.Connection.instance() - consumer = rpc.AdapterConsumer(connection=conn, - topic=FLAGS.cloud_topic, - proxy=controllers['Cloud']) - io_inst = ioloop.IOLoop.instance() - _injected = consumer.attach_to_tornado(io_inst) - http_server = httpserver.HTTPServer(_app) http_server.listen(FLAGS.cc_port) logging.debug('Started HTTP server on %s', FLAGS.cc_port) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 30634429..8e2beb1e 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -45,7 +45,6 @@ from nova.volume import service FLAGS = flags.FLAGS -flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') def _gen_key(user_id, key_name): diff --git a/nova/flags.py b/nova/flags.py index e3feb252..f46017f7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -168,7 +168,6 @@ def DECLARE(name, module_string, flag_values=FLAGS): DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') -#DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on') DEFINE_string('network_topic', 'network', 'the topic network nodes listen on') diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 3501771c..900ff5a9 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -47,10 +47,6 @@ class CloudTestCase(test.BaseTestCase): # set up our cloud self.cloud = cloud.CloudController() - self.cloud_consumer = rpc.AdapterConsumer(connection=self.conn, - topic=FLAGS.cloud_topic, - proxy=self.cloud) - self.injected.append(self.cloud_consumer.attach_to_tornado(self.ioloop)) # set up a service self.compute = service.ComputeService() From 6e85dbe77f3ca37702e506359a6c49af1afb9867 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 18 Aug 2010 18:32:08 -0700 Subject: [PATCH 028/113] fix report state --- nova/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/models.py b/nova/models.py index 70010eab..d0b66d9b 100644 --- a/nova/models.py +++ b/nova/models.py @@ -143,7 +143,7 @@ class Daemon(Base, NovaBase): id = Column(Integer, primary_key=True) node_name = Column(String(255)) #, ForeignKey('physical_node.id')) binary = Column(String(255)) - report_count = Column(Integer) + report_count = Column(Integer, nullable=False, default=0) @classmethod def find_by_args(cls, node_name, binary): From 0061cd7777b4b120699ab9453e8408b7c41a2b33 Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 19 Aug 2010 11:12:44 +0200 Subject: [PATCH 029/113] this file isn't being used --- nova/auth.py | 741 --------------------------------------------------- 1 file changed, 741 deletions(-) delete mode 100644 nova/auth.py diff --git a/nova/auth.py b/nova/auth.py deleted file mode 100644 index 199a887e..00000000 --- a/nova/auth.py +++ /dev/null @@ -1,741 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Nova authentication management -""" - -import logging -import os -import shutil -import string -import tempfile -import uuid -import zipfile - -from nova import crypto -from nova import exception -from nova import flags -from nova import utils -from nova.auth import signer -from nova.network import vpn -from nova.models import User - -#unused imports -#from nova import datastore -#from nova.auth import ldapdriver # for flags -#from nova import objectstore # for flags - -FLAGS = flags.FLAGS - -# NOTE(vish): a user with one of these roles will be a superuser and -# have access to all api commands -flags.DEFINE_list('superuser_roles', ['cloudadmin'], - 'Roles that ignore rbac checking completely') - -# NOTE(vish): a user with one of these roles will have it for every -# project, even if he or she is not a member of the project -flags.DEFINE_list('global_roles', ['cloudadmin', 'itsec'], - 'Roles that apply to all projects') - - -flags.DEFINE_string('credentials_template', - utils.abspath('auth/novarc.template'), - 'Template for creating users rc file') -flags.DEFINE_string('vpn_client_template', - utils.abspath('cloudpipe/client.ovpn.template'), - 'Template for creating users vpn file') -flags.DEFINE_string('credential_vpn_file', 'nova-vpn.conf', - 'Filename of certificate in credentials zip') -flags.DEFINE_string('credential_key_file', 'pk.pem', - 'Filename of private key in credentials zip') -flags.DEFINE_string('credential_cert_file', 'cert.pem', - 'Filename of certificate in credentials zip') -flags.DEFINE_string('credential_rc_file', 'novarc', - 'Filename of rc in credentials zip') - -flags.DEFINE_string('credential_cert_subject', - '/C=US/ST=California/L=MountainView/O=AnsoLabs/' - 'OU=NovaDev/CN=%s-%s', - 'Subject for certificate for users') - -flags.DEFINE_string('auth_driver', 'nova.auth.ldapdriver.FakeLdapDriver', - 'Driver that auth manager uses') - -class AuthBase(object): - """Base class for objects relating to auth - - Objects derived from this class should be stupid data objects with - an id member. They may optionally contain methods that delegate to - AuthManager, but should not implement logic themselves. - """ - @classmethod - def safe_id(cls, obj): - """Safe get object id - - This method will return the id of the object if the object - is of this class, otherwise it will return the original object. - This allows methods to accept objects or ids as paramaters. - - """ - if isinstance(obj, cls): - return obj.id - else: - return obj - - -# anthony - the User class has moved to nova.models -#class User(AuthBase): -# """Object representing a user""" -# def __init__(self, id, name, access, secret, admin): -# AuthBase.__init__(self) -# self.id = id -# self.name = name -# self.access = access -# self.secret = secret -# self.admin = admin -# -# def is_superuser(self): -# return AuthManager().is_superuser(self) -# -# def is_admin(self): -# return AuthManager().is_admin(self) -# -# def has_role(self, role): -# return AuthManager().has_role(self, role) -# -# def add_role(self, role): -# return AuthManager().add_role(self, role) -# -# def remove_role(self, role): -# return AuthManager().remove_role(self, role) -# -# def is_project_member(self, project): -# return AuthManager().is_project_member(self, project) -# -# def is_project_manager(self, project): -# return AuthManager().is_project_manager(self, project) -# -# def generate_key_pair(self, name): -# return AuthManager().generate_key_pair(self.id, name) -# -# def create_key_pair(self, name, public_key, fingerprint): -# return AuthManager().create_key_pair(self.id, -# name, -# public_key, -# fingerprint) -# -# def get_key_pair(self, name): -# return AuthManager().get_key_pair(self.id, name) -# -# def delete_key_pair(self, name): -# return AuthManager().delete_key_pair(self.id, name) -# -# def get_key_pairs(self): -# return AuthManager().get_key_pairs(self.id) -# -# def __repr__(self): -# return "User('%s', '%s', '%s', '%s', %s)" % (self.id, -# self.name, -# self.access, -# self.secret, -# self.admin) - - -class KeyPair(AuthBase): - """Represents an ssh key returned from the datastore - - Even though this object is named KeyPair, only the public key and - fingerprint is stored. The user's private key is not saved. - """ - def __init__(self, id, name, owner_id, public_key, fingerprint): - AuthBase.__init__(self) - self.id = id - self.name = name - self.owner_id = owner_id - self.public_key = public_key - self.fingerprint = fingerprint - - def __repr__(self): - return "KeyPair('%s', '%s', '%s', '%s', '%s')" % (self.id, - self.name, - self.owner_id, - self.public_key, - self.fingerprint) - - -class Project(AuthBase): - """Represents a Project returned from the datastore""" - def __init__(self, id, name, project_manager_id, description, member_ids): - AuthBase.__init__(self) - self.id = id - self.name = name - self.project_manager_id = project_manager_id - self.description = description - self.member_ids = member_ids - - @property - def project_manager(self): - return AuthManager().get_user(self.project_manager_id) - - @property - def vpn_ip(self): - ip, port = AuthManager().get_project_vpn_data(self) - return ip - - @property - def vpn_port(self): - ip, port = AuthManager().get_project_vpn_data(self) - return port - - def has_manager(self, user): - return AuthManager().is_project_manager(user, self) - - def has_member(self, user): - return AuthManager().is_project_member(user, self) - - def add_role(self, user, role): - return AuthManager().add_role(user, role, self) - - def remove_role(self, user, role): - return AuthManager().remove_role(user, role, self) - - def has_role(self, user, role): - return AuthManager().has_role(user, role, self) - - def get_credentials(self, user): - return AuthManager().get_credentials(user, self) - - def __repr__(self): - return "Project('%s', '%s', '%s', '%s', %s)" % (self.id, - self.name, - self.project_manager_id, - self.description, - self.member_ids) - - - -class AuthManager(object): - """Manager Singleton for dealing with Users, Projects, and Keypairs - - Methods accept objects or ids. - - AuthManager uses a driver object to make requests to the data backend. - See ldapdriver for reference. - - AuthManager also manages associated data related to Auth objects that - need to be more accessible, such as vpn ips and ports. - """ - _instance = None - def __new__(cls, *args, **kwargs): - """Returns the AuthManager singleton""" - if not cls._instance: - cls._instance = super(AuthManager, cls).__new__(cls) - return cls._instance - - def __init__(self, driver=None, *args, **kwargs): - """Inits the driver from parameter or flag - - __init__ is run every time AuthManager() is called, so we only - reset the driver if it is not set or a new driver is specified. - """ - if driver or not getattr(self, 'driver', None): - self.driver = utils.import_class(driver or FLAGS.auth_driver) - - def authenticate(self, access, signature, params, verb='GET', - server_string='127.0.0.1:8773', path='/', - check_type='ec2', headers=None): - """Authenticates AWS request using access key and signature - - If the project is not specified, attempts to authenticate to - a project with the same name as the user. This way, older tools - that have no project knowledge will still work. - - @type access: str - @param access: Access key for user in the form "access:project". - - @type signature: str - @param signature: Signature of the request. - - @type params: list of str - @param params: Web paramaters used for the signature. - - @type verb: str - @param verb: Web request verb ('GET' or 'POST'). - - @type server_string: str - @param server_string: Web request server string. - - @type path: str - @param path: Web request path. - - @type check_type: str - @param check_type: Type of signature to check. 'ec2' for EC2, 's3' for - S3. Any other value will cause signature not to be - checked. - - @type headers: list - @param headers: HTTP headers passed with the request (only needed for - s3 signature checks) - - @rtype: tuple (User, Project) - @return: User and project that the request represents. - """ - # TODO(vish): check for valid timestamp - (access_key, sep, project_id) = access.partition(':') - - logging.info('Looking up user: %r', access_key) - user = self.get_user_from_access_key(access_key) - logging.info('user: %r', user) - if user == None: - raise exception.NotFound('No user found for access key %s' % - access_key) - - # NOTE(vish): if we stop using project name as id we need better - # logic to find a default project for user - if project_id is '': - project_id = user.name - - project = self.get_project(project_id) - if project == None: - raise exception.NotFound('No project called %s could be found' % - project_id) - if not self.is_admin(user) and not self.is_project_member(user, - project): - raise exception.NotFound('User %s is not a member of project %s' % - (user.id, project.id)) - if check_type == 's3': - expected_signature = signer.Signer(user.secret.encode()).s3_authorization(headers, verb, path) - logging.debug('user.secret: %s', user.secret) - logging.debug('expected_signature: %s', expected_signature) - logging.debug('signature: %s', signature) - if signature != expected_signature: - raise exception.NotAuthorized('Signature does not match') - elif check_type == 'ec2': - # NOTE(vish): hmac can't handle unicode, so encode ensures that - # secret isn't unicode - expected_signature = signer.Signer(user.secret.encode()).generate( - params, verb, server_string, path) - logging.debug('user.secret: %s', user.secret) - logging.debug('expected_signature: %s', expected_signature) - logging.debug('signature: %s', signature) - if signature != expected_signature: - raise exception.NotAuthorized('Signature does not match') - return (user, project) - - def get_access_key(self, user, project): - """Get an access key that includes user and project""" - if not isinstance(user, User): - user = self.get_user(user) - return "%s:%s" % (user.access, Project.safe_id(project)) - - def is_superuser(self, user): - """Checks for superuser status, allowing user to bypass rbac - - @type user: User or uid - @param user: User to check. - - @rtype: bool - @return: True for superuser. - """ - if not isinstance(user, User): - user = self.get_user(user) - # NOTE(vish): admin flag on user represents superuser - if user.admin: - return True - for role in FLAGS.superuser_roles: - if self.has_role(user, role): - return True - - def is_admin(self, user): - """Checks for admin status, allowing user to access all projects - - @type user: User or uid - @param user: User to check. - - @rtype: bool - @return: True for admin. - """ - if not isinstance(user, User): - user = self.get_user(user) - if self.is_superuser(user): - return True - for role in FLAGS.global_roles: - if self.has_role(user, role): - return True - - def has_role(self, user, role, project=None): - """Checks existence of role for user - - If project is not specified, checks for a global role. If project - is specified, checks for the union of the global role and the - project role. - - Role 'projectmanager' only works for projects and simply checks to - see if the user is the project_manager of the specified project. It - is the same as calling is_project_manager(user, project). - - @type user: User or uid - @param user: User to check. - - @type role: str - @param role: Role to check. - - @type project: Project or project_id - @param project: Project in which to look for local role. - - @rtype: bool - @return: True if the user has the role. - """ - with self.driver() as drv: - if role == 'projectmanager': - if not project: - raise exception.Error("Must specify project") - return self.is_project_manager(user, project) - - global_role = drv.has_role(User.safe_id(user), - role, - None) - if not global_role: - return global_role - - if not project or role in FLAGS.global_roles: - return global_role - - return drv.has_role(User.safe_id(user), - role, - Project.safe_id(project)) - - def add_role(self, user, role, project=None): - """Adds role for user - - If project is not specified, adds a global role. If project - is specified, adds a local role. - - The 'projectmanager' role is special and can't be added or removed. - - @type user: User or uid - @param user: User to which to add role. - - @type role: str - @param role: Role to add. - - @type project: Project or project_id - @param project: Project in which to add local role. - """ - with self.driver() as drv: - drv.add_role(User.safe_id(user), role, Project.safe_id(project)) - - def remove_role(self, user, role, project=None): - """Removes role for user - - If project is not specified, removes a global role. If project - is specified, removes a local role. - - The 'projectmanager' role is special and can't be added or removed. - - @type user: User or uid - @param user: User from which to remove role. - - @type role: str - @param role: Role to remove. - - @type project: Project or project_id - @param project: Project in which to remove local role. - """ - with self.driver() as drv: - drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) - - def get_project(self, pid): - """Get project object by id""" - with self.driver() as drv: - project_dict = drv.get_project(pid) - if project_dict: - return Project(**project_dict) - - def get_projects(self, user=None): - """Retrieves list of projects, optionally filtered by user""" - with self.driver() as drv: - project_list = drv.get_projects(User.safe_id(user)) - if not project_list: - return [] - return [Project(**project_dict) for project_dict in project_list] - - def create_project(self, name, manager_user, - description=None, member_users=None): - """Create a project - - @type name: str - @param name: Name of the project to create. The name will also be - used as the project id. - - @type manager_user: User or uid - @param manager_user: This user will be the project manager. - - @type description: str - @param project: Description of the project. If no description is - specified, the name of the project will be used. - - @type member_users: list of User or uid - @param: Initial project members. The project manager will always be - added as a member, even if he isn't specified in this list. - - @rtype: Project - @return: The new project. - """ - if member_users: - member_users = [User.safe_id(u) for u in member_users] - with self.driver() as drv: - project_dict = drv.create_project(name, - User.safe_id(manager_user), - description, - member_users) - if project_dict: - return Project(**project_dict) - - def add_to_project(self, user, project): - """Add user to project""" - with self.driver() as drv: - return drv.add_to_project(User.safe_id(user), - Project.safe_id(project)) - - def is_project_manager(self, user, project): - """Checks if user is project manager""" - if not isinstance(project, Project): - project = self.get_project(project) - return User.safe_id(user) == project.project_manager_id - - def is_project_member(self, user, project): - """Checks to see if user is a member of project""" - if not isinstance(project, Project): - project = self.get_project(project) - return User.safe_id(user) in project.member_ids - - def remove_from_project(self, user, project): - """Removes a user from a project""" - with self.driver() as drv: - return drv.remove_from_project(User.safe_id(user), - Project.safe_id(project)) - - def get_project_vpn_data(self, project): - """Gets vpn ip and port for project - - @type project: Project or project_id - @param project: Project from which to get associated vpn data - - @rvalue: tuple of (str, str) - @return: A tuple containing (ip, port) or None, None if vpn has - not been allocated for user. - """ - network_data = vpn.NetworkData.lookup(Project.safe_id(project)) - if not network_data: - raise exception.NotFound('project network data has not been set') - return (network_data.ip, network_data.port) - - def delete_project(self, project): - """Deletes a project""" - with self.driver() as drv: - return drv.delete_project(Project.safe_id(project)) - - def get_user(self, uid): - """Retrieves a user by id""" - with self.driver() as drv: - user_dict = drv.get_user(uid) - if user_dict: - return User(**user_dict) - - def get_user_from_access_key(self, access_key): - """Retrieves a user by access key""" - with self.driver() as drv: - user_dict = drv.get_user_from_access_key(access_key) - if user_dict: - return User(**user_dict) - - def get_users(self): - """Retrieves a list of all users""" - with self.driver() as drv: - user_list = drv.get_users() - if not user_list: - return [] - return [User(**user_dict) for user_dict in user_list] - - def create_user(self, name, access=None, secret=None, admin=False): - """Creates a user - - @type name: str - @param name: Name of the user to create. - - @type access: str - @param access: Access Key (defaults to a random uuid) - - @type secret: str - @param secret: Secret Key (defaults to a random uuid) - - @type admin: bool - @param admin: Whether to set the admin flag. The admin flag gives - superuser status regardless of roles specifed for the user. - - @type create_project: bool - @param: Whether to create a project for the user with the same name. - - @rtype: User - @return: The new user. - """ - if access == None: access = str(uuid.uuid4()) - if secret == None: secret = str(uuid.uuid4()) - with self.driver() as drv: - user_dict = drv.create_user(name, access, secret, admin) - if user_dict: - return User(**user_dict) - - def delete_user(self, user): - """Deletes a user""" - with self.driver() as drv: - drv.delete_user(User.safe_id(user)) - - def generate_key_pair(self, user, key_name): - """Generates a key pair for a user - - Generates a public and private key, stores the public key using the - key_name, and returns the private key and fingerprint. - - @type user: User or uid - @param user: User for which to create key pair. - - @type key_name: str - @param key_name: Name to use for the generated KeyPair. - - @rtype: tuple (private_key, fingerprint) - @return: A tuple containing the private_key and fingerprint. - """ - # NOTE(vish): generating key pair is slow so check for legal - # creation before creating keypair - uid = User.safe_id(user) - with self.driver() as drv: - if not drv.get_user(uid): - raise exception.NotFound("User %s doesn't exist" % user) - if drv.get_key_pair(uid, key_name): - raise exception.Duplicate("The keypair %s already exists" - % key_name) - private_key, public_key, fingerprint = crypto.generate_key_pair() - self.create_key_pair(uid, key_name, public_key, fingerprint) - return private_key, fingerprint - - def create_key_pair(self, user, key_name, public_key, fingerprint): - """Creates a key pair for user""" - with self.driver() as drv: - kp_dict = drv.create_key_pair(User.safe_id(user), - key_name, - public_key, - fingerprint) - if kp_dict: - return KeyPair(**kp_dict) - - def get_key_pair(self, user, key_name): - """Retrieves a key pair for user""" - with self.driver() as drv: - kp_dict = drv.get_key_pair(User.safe_id(user), key_name) - if kp_dict: - return KeyPair(**kp_dict) - - def get_key_pairs(self, user): - """Retrieves all key pairs for user""" - with self.driver() as drv: - kp_list = drv.get_key_pairs(User.safe_id(user)) - if not kp_list: - return [] - return [KeyPair(**kp_dict) for kp_dict in kp_list] - - def delete_key_pair(self, user, key_name): - """Deletes a key pair for user""" - with self.driver() as drv: - drv.delete_key_pair(User.safe_id(user), key_name) - - def get_credentials(self, user, project=None): - """Get credential zip for user in project""" - if not isinstance(user, User): - user = self.get_user(user) - if project is None: - project = user.id - pid = Project.safe_id(project) - rc = self.__generate_rc(user.access, user.secret, pid) - private_key, signed_cert = self._generate_x509_cert(user.id, pid) - - tmpdir = tempfile.mkdtemp() - zf = os.path.join(tmpdir, "temp.zip") - zippy = zipfile.ZipFile(zf, 'w') - zippy.writestr(FLAGS.credential_rc_file, rc) - zippy.writestr(FLAGS.credential_key_file, private_key) - zippy.writestr(FLAGS.credential_cert_file, signed_cert) - - network_data = vpn.NetworkData.lookup(pid) - if network_data: - configfile = open(FLAGS.vpn_client_template,"r") - s = string.Template(configfile.read()) - configfile.close() - config = s.substitute(keyfile=FLAGS.credential_key_file, - certfile=FLAGS.credential_cert_file, - ip=network_data.ip, - port=network_data.port) - zippy.writestr(FLAGS.credential_vpn_file, config) - else: - logging.warn("No vpn data for project %s" % - pid) - - zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(user.id)) - zippy.close() - with open(zf, 'rb') as f: - buffer = f.read() - - shutil.rmtree(tmpdir) - return buffer - - def get_environment_rc(self, user, project=None): - """Get credential zip for user in project""" - if not isinstance(user, User): - user = self.get_user(user) - if project is None: - project = user.id - pid = Project.safe_id(project) - return self.__generate_rc(user.access, user.secret, pid) - - def __generate_rc(self, access, secret, pid): - """Generate rc file for user""" - rc = open(FLAGS.credentials_template).read() - rc = rc % { 'access': access, - 'project': pid, - 'secret': secret, - 'ec2': FLAGS.ec2_url, - 's3': 'http://%s:%s' % (FLAGS.s3_host, FLAGS.s3_port), - 'nova': FLAGS.ca_file, - 'cert': FLAGS.credential_cert_file, - 'key': FLAGS.credential_key_file, - } - return rc - - def _generate_x509_cert(self, uid, pid): - """Generate x509 cert for user""" - (private_key, csr) = crypto.generate_x509_cert( - self.__cert_subject(uid)) - # TODO(joshua): This should be async call back to the cloud controller - signed_cert = crypto.sign_csr(csr, pid) - return (private_key, signed_cert) - - def __cert_subject(self, uid): - """Helper to generate cert subject""" - return FLAGS.credential_cert_subject % (uid, utils.isotime()) From 83d3755cff07ca24d7933e1159f83610bafe4d4a Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 19 Aug 2010 12:28:45 +0200 Subject: [PATCH 030/113] Data abstraction for compute service --- nova/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/models.py b/nova/models.py index d0b66d9b..ea529713 100644 --- a/nova/models.py +++ b/nova/models.py @@ -100,6 +100,12 @@ class NovaBase(object): session = NovaBase.get_session() session.refresh(self) + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + class Image(Base, NovaBase): __tablename__ = 'images' From 959f78889b3f69e90b4d67735b6a43c0a35dbf0f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 19 Aug 2010 13:58:43 -0700 Subject: [PATCH 031/113] move volume code into datalayer and cleanup --- nova/models.py | 1 - nova/tests/volume_unittest.py | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/models.py b/nova/models.py index ea529713..ef10398e 100644 --- a/nova/models.py +++ b/nova/models.py @@ -231,7 +231,6 @@ class Instance(Base, NovaBase): class Volume(Base, NovaBase): __tablename__ = 'volumes' id = Column(Integer, primary_key=True) - volume_id = Column(String(255)) user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) project_id = Column(String(255)) #, ForeignKey('projects.id')) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 82f71901..90cd04c6 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -21,6 +21,7 @@ import logging from twisted.internet import defer from nova import exception +from nova import db from nova import flags from nova import models from nova import test @@ -89,7 +90,7 @@ class VolumeTestCase(test.TrialTestCase): self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), - volume_service.NoMoreBlades) + db.sqlalchemy.api.NoMoreBlades) for id in vols: yield self.volume.delete_volume(id) @@ -102,23 +103,21 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' mountpoint = "/dev/sdf" volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) - vol = models.Volume.find(volume_id) - self.volume.start_attach(volume_id, instance_id, mountpoint) if FLAGS.fake_tests: - self.volume.finish_attach(volume_id) + db.volume_attached(None, volume_id, instance_id, mountpoint) else: rv = yield self.compute.attach_volume(instance_id, volume_id, mountpoint) + vol = db.volume_get(None, volume_id) self.assertEqual(vol.status, "in-use") self.assertEqual(vol.attach_status, "attached") self.assertEqual(vol.instance_id, instance_id) self.assertEqual(vol.mountpoint, mountpoint) self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) - self.volume.start_detach(volume_id) if FLAGS.fake_tests: - self.volume.finish_detach(volume_id) + db.volume_detached(None, volume_id) else: rv = yield self.volume.detach_volume(instance_id, volume_id) From 4c0e4fd0d72a93b7ba4cdcdb04756f4592b31bed Mon Sep 17 00:00:00 2001 From: andy Date: Sat, 21 Aug 2010 12:47:21 +0200 Subject: [PATCH 032/113] Alphabetize the methods in the db layer. There are enough of them in there that it is probably useful to keep them organized. Also moved the NoMoreBlades to db, it is likely to be shared by any implementation. --- nova/models.py | 12 ++++++------ nova/tests/volume_unittest.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/models.py b/nova/models.py index ef10398e..e4cd3733 100644 --- a/nova/models.py +++ b/nova/models.py @@ -179,7 +179,7 @@ class Instance(Base, NovaBase): def project(self): return auth.manager.AuthManager().get_project(self.project_id) - # FIXME: make this opaque somehow + # TODO(vish): make this opaque somehow @property def name(self): return "i-%s" % self.id @@ -237,12 +237,12 @@ class Volume(Base, NovaBase): node_name = Column(String(255)) #, ForeignKey('physical_node.id')) size = Column(Integer) - alvailability_zone = Column(String(255)) # FIXME foreign key? + availability_zone = Column(String(255)) # TODO(vish) foreign key? instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) mountpoint = Column(String(255)) - attach_time = Column(String(255)) # FIXME datetime - status = Column(String(255)) # FIXME enum? - attach_status = Column(String(255)) # FIXME enum + attach_time = Column(String(255)) # TODO(vish) datetime + status = Column(String(255)) # TODO(vish) enum? + attach_status = Column(String(255)) # TODO(vish) enum class ExportDevice(Base, NovaBase): __tablename__ = 'export_devices' @@ -254,7 +254,7 @@ class ExportDevice(Base, NovaBase): uselist=False)) -#FIXME can these both come from the same baseclass? +# TODO(vish): can these both come from the same baseclass? class FixedIp(Base, NovaBase): __tablename__ = 'fixed_ips' id = Column(Integer, primary_key=True) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 90cd04c6..37ee6c72 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -90,7 +90,7 @@ class VolumeTestCase(test.TrialTestCase): self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), - db.sqlalchemy.api.NoMoreBlades) + db.NoMoreBlades) for id in vols: yield self.volume.delete_volume(id) From cfc320ae5ac4cc1799282cee309de93095e5b2ec Mon Sep 17 00:00:00 2001 From: andy Date: Sat, 21 Aug 2010 14:10:36 +0200 Subject: [PATCH 033/113] Add db abstraction and unittets for service.py. Also cleans up some style pieces. --- nova/tests/service_unittest.py | 139 +++++++++++++++++++++++++++++++++ run_tests.py | 1 + 2 files changed, 140 insertions(+) create mode 100644 nova/tests/service_unittest.py diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py new file mode 100644 index 00000000..44949420 --- /dev/null +++ b/nova/tests/service_unittest.py @@ -0,0 +1,139 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit Tests for remote procedure calls using queue +""" + +import logging + +import mox +from twisted.internet import defer + +from nova import exception +from nova import flags +from nova import rpc +from nova import test +from nova import service + + +FLAGS = flags.FLAGS + + +class ServiceTestCase(test.BaseTestCase): + """Test cases for rpc""" + def setUp(self): # pylint: disable=C0103 + super(ServiceTestCase, self).setUp() + self.mox.StubOutWithMock(service, 'db') + + def test_create(self): + self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) + rpc.AdapterConsumer(connection=mox.IgnoreArg(), + topic='run_tests.py', + proxy=mox.IsA(service.Service) + ).AndReturn(rpc.AdapterConsumer) + + rpc.AdapterConsumer(connection=mox.IgnoreArg(), + topic='run_tests.py.%s' % FLAGS.node_name, + proxy=mox.IsA(service.Service) + ).AndReturn(rpc.AdapterConsumer) + rpc.AdapterConsumer.attach_to_twisted() + rpc.AdapterConsumer.attach_to_twisted() + self.mox.ReplayAll() + + app = service.Service.create() + self.assert_(app) + + # We're testing sort of weird behavior in how report_state decides + # whether it is disconnected, it looks for a variable on itself called + # 'model_disconnected' and report_state doesn't really do much so this + # these are mostly just for coverage + + def test_report_state(self): + node_name = 'foo' + binary = 'bar' + daemon_ref = {'node_name': node_name, + 'binary': binary, + 'report_count': 0 + } + + service.db.daemon_get(None, node_name, binary).AndReturn(daemon_ref) + service.db.daemon_update(None, node_name, binary, + mox.ContainsKeyValue('report_count', 1)) + + self.mox.ReplayAll() + s = service.Service() + rv = yield s.report_state(node_name, binary) + + + def test_report_state_no_daemon(self): + node_name = 'foo' + binary = 'bar' + daemon_ref = {'node_name': node_name, + 'binary': binary, + 'report_count': 0 + } + + service.db.daemon_get(None, node_name, binary).AndRaise( + exception.NotFound()) + service.db.daemon_create(None, daemon_ref).AndReturn(daemon_ref) + service.db.daemon_update(None, node_name, binary, + mox.ContainsKeyValue('report_count', 1)) + + self.mox.ReplayAll() + s = service.Service() + rv = yield s.report_state(node_name, binary) + + + def test_report_state_newly_disconnected(self): + node_name = 'foo' + binary = 'bar' + daemon_ref = {'node_name': node_name, + 'binary': binary, + 'report_count': 0 + } + + service.db.daemon_get(None, node_name, binary).AndRaise( + Exception()) + + self.mox.ReplayAll() + s = service.Service() + rv = yield s.report_state(node_name, binary) + + self.assert_(s.model_disconnected) + + + def test_report_state_newly_connected(self): + node_name = 'foo' + binary = 'bar' + daemon_ref = {'node_name': node_name, + 'binary': binary, + 'report_count': 0 + } + + service.db.daemon_get(None, node_name, binary).AndReturn(daemon_ref) + service.db.daemon_update(None, node_name, binary, + mox.ContainsKeyValue('report_count', 1)) + + self.mox.ReplayAll() + s = service.Service() + s.model_disconnected = True + rv = yield s.report_state(node_name, binary) + + self.assert_(not s.model_disconnected) + diff --git a/run_tests.py b/run_tests.py index 82c1aa9c..c47cbe2e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -60,6 +60,7 @@ from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * from nova.tests.rpc_unittest import * +from nova.tests.service_unittest import * from nova.tests.validator_unittest import * from nova.tests.volume_unittest import * From 44a796c99868c13d5eb43f60908b30f78bbd4290 Mon Sep 17 00:00:00 2001 From: andy Date: Sat, 21 Aug 2010 15:37:00 +0200 Subject: [PATCH 034/113] Moves auth.manager to the data layer. A couple weird things are going on, I added a try-except in Manager.delete_project because it seems to have an issue finding the network to delete, I think something is probably deleting it before the tests get a chance to. Also stubbed out task.LoopingCall in service_unittest because there wasn't a good way to kill the task from outside of service.Service.create() --- nova/auth/manager.py | 37 ++++++++++++++++++---------------- nova/tests/network_unittest.py | 4 ++++ nova/tests/service_unittest.py | 11 ++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index eed67d8c..070c5508 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -29,6 +29,7 @@ import uuid import zipfile from nova import crypto +from nova import db from nova import exception from nova import flags from nova import models @@ -202,11 +203,6 @@ class Project(AuthBase): ip, port = AuthManager().get_project_vpn_data(self) return port - @property - def network(self): - session = models.create_session() - return session.query(models.Network).filter_by(project_id=self.id).first() - def has_manager(self, user): return AuthManager().is_project_manager(user, self) @@ -498,8 +494,8 @@ class AuthManager(object): return [] return [Project(**project_dict) for project_dict in project_list] - def create_project(self, name, manager_user, - description=None, member_users=None): + def create_project(self, name, manager_user, description=None, + member_users=None, context=None): """Create a project @type name: str @@ -530,8 +526,7 @@ class AuthManager(object): if project_dict: project = Project(**project_dict) # FIXME(ja): EVIL HACK - net = models.Network(project_id=project.id) - net.save() + db.network_create(context, {'project_id': project.id}) return project def add_to_project(self, user, project): @@ -558,7 +553,7 @@ class AuthManager(object): return drv.remove_from_project(User.safe_id(user), Project.safe_id(project)) - def get_project_vpn_data(self, project): + def get_project_vpn_data(self, project, context=None): """Gets vpn ip and port for project @type project: Project or project_id @@ -571,19 +566,27 @@ class AuthManager(object): # FIXME(vish): this shouldn't be messing with the datamodel directly if not isinstance(project, Project): project = self.get_project(project) - if not project.network.vpn_public_port: - raise exception.NotFound('project network data has not been set') - return (project.network.vpn_public_ip_str, - project.network.vpn_public_port) + + network_ref = db.project_get_network(context, project.id) - def delete_project(self, project): + if not network_ref['vpn_public_port']: + raise exception.NotFound('project network data has not been set') + return (network_ref['vpn_public_ip_str'], + network_ref['vpn_public_port']) + + def delete_project(self, project, context=None): """Deletes a project""" # FIXME(ja): EVIL HACK if not isinstance(project, Project): project = self.get_project(project) - project.network.delete() + network_ref = db.project_get_network(context, project.id) + try: + db.network_destroy(context, network_ref['id']) + except: + logging.exception('Could not destroy network: %s', + network_ref['id']) with self.driver() as drv: - return drv.delete_project(Project.safe_id(project)) + drv.delete_project(Project.safe_id(project)) def get_user(self, uid): """Retrieves a user by id""" diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 0f2ce060..76c76edb 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -67,6 +67,8 @@ class NetworkTestCase(test.TrialTestCase): def tearDown(self): # pylint: disable=C0103 super(NetworkTestCase, self).tearDown() + # TODO(termie): this should really be instantiating clean datastores + # in between runs, one failure kills all the tests for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) @@ -275,6 +277,8 @@ def is_allocated_in_project(address, project_id): fixed_ip = models.FixedIp.find_by_ip_str(address) project_net = service.get_network_for_project(project_id) # instance exists until release + logging.error('fixed_ip.instance: %s', fixed_ip.instance) + logging.error('project_net: %s', project_net) return fixed_ip.instance is not None and fixed_ip.network == project_net diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 44949420..48298846 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -43,6 +43,8 @@ class ServiceTestCase(test.BaseTestCase): def test_create(self): self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) + self.mox.StubOutWithMock( + service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic='run_tests.py', proxy=mox.IsA(service.Service) @@ -52,6 +54,15 @@ class ServiceTestCase(test.BaseTestCase): topic='run_tests.py.%s' % FLAGS.node_name, proxy=mox.IsA(service.Service) ).AndReturn(rpc.AdapterConsumer) + + # Stub out looping call a bit needlessly since we don't have an easy + # way to cancel it (yet) when the tests finishes + service.task.LoopingCall( + mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn( + service.task.LoopingCall) + service.task.LoopingCall.start(interval=mox.IgnoreArg(), + now=mox.IgnoreArg()) + rpc.AdapterConsumer.attach_to_twisted() rpc.AdapterConsumer.attach_to_twisted() self.mox.ReplayAll() From e64136c381fa71677860713ce646a364ddf36b90 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 23 Aug 2010 13:55:16 -0700 Subject: [PATCH 035/113] Refactored network model access into data abstraction layer. Also changed the name to floating_ip. --- bin/nova-dhcpbridge | 23 +++++------ nova/endpoint/cloud.py | 28 +++++++------- nova/models.py | 8 ++-- nova/tests/network_unittest.py | 70 +++++++++++++++++----------------- 4 files changed, 66 insertions(+), 63 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index b17a56e6..8008100f 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -25,9 +25,9 @@ import logging import os import sys -#TODO(joshua): there is concern that the user dnsmasq runs under will not -# have nova in the path. This should be verified and if it is -# not true the ugly line below can be removed +# TODO(joshua): there is concern that the user dnsmasq runs under will not +# have nova in the path. This should be verified and if it is +# not true the ugly line below can be removed sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) from nova import flags @@ -36,6 +36,7 @@ from nova import utils from nova.network import linux_net from nova.network import service from nova import datastore # for redis_db flag +from nova.auth import manager # for auth flags FLAGS = flags.FLAGS @@ -43,16 +44,16 @@ FLAGS = flags.FLAGS def add_lease(_mac, ip, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: - logging.debug("leasing_ip") + logging.debug("leasing ip") from nova import models print models.FixedIp.count() print models.Network.count() print FLAGS.sql_connection - service.VlanNetworkService().lease_ip(ip) + service.VlanNetworkService().lease_fixed_ip(ip) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), - {"method": "lease_ip", - "args": {"fixed_ip_str": ip}}) + {"method": "lease_fixed_ip", + "args": {"address": ip}}) def old_lease(_mac, _ip, _hostname, _interface): @@ -63,12 +64,12 @@ def old_lease(_mac, _ip, _hostname, _interface): def del_lease(_mac, ip, _hostname, _interface): """Called when a lease expires.""" if FLAGS.fake_rabbit: - logging.debug("releasing_ip") - service.VlanNetworkService().release_ip(ip) + logging.debug("releasing ip") + service.VlanNetworkService().release_fixed_ip(ip) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), - {"method": "release_ip", - "args": {"fixed_ip_str": ip}}) + {"method": "release_fixed_ip", + "args": {"address": ip}}) def init_leases(interface): diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index e5d4661d..e64005c2 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -311,7 +311,7 @@ class CloudController(object): def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py - address = network_model.ElasticIp.lookup(public_ip) + address = network_model.FloatingIp.lookup(public_ip) if address and (context.user.is_admin() or address['project_id'] == context.project.id): return address raise exception.NotFound("Address at ip %s not found" % public_ip) @@ -459,7 +459,7 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in network_model.ElasticIp.all(): + for address in network_model.FloatingIp.all(): # TODO(vish): implement a by_project iterator for addresses if (context.user.is_admin() or address['project_id'] == context.project.id): @@ -481,7 +481,7 @@ class CloudController(object): def allocate_address(self, context, **kwargs): network_topic = yield self._get_network_topic(context) public_ip = yield rpc.call(network_topic, - {"method": "allocate_elastic_ip", + {"method": "allocate_floating_ip", "args": {"user_id": context.user.id, "project_id": context.project.id}}) defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @@ -492,8 +492,8 @@ class CloudController(object): # NOTE(vish): Should we make sure this works? network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "deallocate_elastic_ip", - "args": {"elastic_ip": public_ip}}) + {"method": "deallocate_floating_ip", + "args": {"floating_ip": public_ip}}) defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') @@ -503,8 +503,8 @@ class CloudController(object): address = self._get_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "associate_elastic_ip", - "args": {"elastic_ip": address['address'], + {"method": "associate_floating_ip", + "args": {"floating_ip": address['address'], "fixed_ip": instance['private_dns_name'], "instance_id": instance['instance_id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @@ -515,8 +515,8 @@ class CloudController(object): address = self._get_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "disassociate_elastic_ip", - "args": {"elastic_ip": address['address']}}) + {"method": "disassociate_floating_ip", + "args": {"floating_ip": address['address']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks @@ -617,15 +617,15 @@ class CloudController(object): logging.warning("Instance %s was not found during terminate" % i) continue - elastic_ip = network_model.get_public_ip_for_instance(i) - if elastic_ip: - logging.debug("Disassociating address %s" % elastic_ip) + floating_ip = network_model.get_public_ip_for_instance(i) + if floating_ip: + logging.debug("Disassociating address %s" % floating_ip) # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about # checking this later. Perhaps in the scheduler? rpc.cast(network_topic, - {"method": "disassociate_elastic_ip", - "args": {"elastic_ip": elastic_ip}}) + {"method": "disassociate_floating_ip", + "args": {"floating_ip": floating_ip}}) fixed_ip = instance.get('private_dns_name', None) if fixed_ip: diff --git a/nova/models.py b/nova/models.py index e4cd3733..70caeff7 100644 --- a/nova/models.py +++ b/nova/models.py @@ -278,12 +278,12 @@ class FixedIp(Base, NovaBase): raise exception.NotFound("No model for ip str %s" % ip_str) -class ElasticIp(Base, NovaBase): - __tablename__ = 'elastic_ips' +class FloatingIp(Base, NovaBase): + __tablename__ = 'floating_ips' id = Column(Integer, primary_key=True) ip_str = Column(String(255), unique=True) fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) - fixed_ip = relationship(FixedIp, backref=backref('elastic_ips')) + fixed_ip = relationship(FixedIp, backref=backref('floating_ips')) project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) node_name = Column(String(255)) #, ForeignKey('physical_node.id')) @@ -305,7 +305,7 @@ class Network(Base, NovaBase): kind = Column(String(255)) injected = Column(Boolean, default=False) - network_str = Column(String(255)) + cidr = Column(String(255)) netmask = Column(String(255)) bridge = Column(String(255)) gateway = Column(String(255)) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 76c76edb..c4c49621 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -21,8 +21,8 @@ Unit Tests for network code import IPy import os import logging -import tempfile +from nova import db from nova import exception from nova import flags from nova import models @@ -30,7 +30,6 @@ from nova import test from nova import utils from nova.auth import manager from nova.network import service -from nova.network.exception import NoMoreAddresses, NoMoreNetworks FLAGS = flags.FLAGS @@ -59,49 +58,52 @@ class NetworkTestCase(test.TrialTestCase): name)) # create the necessary network data for the project self.service.set_network_host(self.projects[i].id) - instance = models.Instance() - instance.mac_address = utils.generate_mac() - instance.hostname = 'fake' - instance.save() - self.instance_id = instance.id + instance_id = db.instance_create(None, + {'mac_address': utils.generate_mac()}) + self.instance_id = instance_id + instance_id = db.instance_create(None, + {'mac_address': utils.generate_mac()}) + self.instance2_id = instance_id def tearDown(self): # pylint: disable=C0103 super(NetworkTestCase, self).tearDown() # TODO(termie): this should really be instantiating clean datastores # in between runs, one failure kills all the tests + db.instance_destroy(None, self.instance_id) + db.instance_destroy(None, self.instance2_id) for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" - # FIXME better way of adding elastic ips + # TODO(vish): better way of adding floating ips pubnet = IPy.IP(flags.FLAGS.public_range) ip_str = str(pubnet[0]) try: - elastic_ip = models.ElasticIp.find_by_ip_str(ip_str) + floating_ip = models.FloatingIp.find_by_ip_str(ip_str) except exception.NotFound: - elastic_ip = models.ElasticIp() - elastic_ip.ip_str = ip_str - elastic_ip.node_name = FLAGS.node_name - elastic_ip.save() - eaddress = self.service.allocate_elastic_ip(self.projects[0].id) + floating_ip = models.FloatingIp() + floating_ip.ip_str = ip_str + floating_ip.node_name = FLAGS.node_name + floating_ip.save() + eaddress = self.service.allocate_floating_ip(self.projects[0].id) faddress = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) self.assertEqual(eaddress, str(pubnet[0])) - self.service.associate_elastic_ip(eaddress, faddress) + self.service.associate_floating_ip(eaddress, faddress) # FIXME datamodel abstraction - self.assertEqual(elastic_ip.fixed_ip.ip_str, faddress) - self.service.disassociate_elastic_ip(eaddress) - self.assertEqual(elastic_ip.fixed_ip, None) - self.service.deallocate_elastic_ip(eaddress) + self.assertEqual(floating_ip.fixed_ip.ip_str, faddress) + self.service.disassociate_floating_ip(eaddress) + self.assertEqual(floating_ip.fixed_ip, None) + self.service.deallocate_floating_ip(eaddress) self.service.deallocate_fixed_ip(faddress) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" address = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) - net = service.get_network_for_project(self.projects[0].id) + net = db.project_get_network(None, self.projects[0].id) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) issue_ip(address, net.bridge) self.service.deallocate_fixed_ip(address) @@ -117,10 +119,10 @@ class NetworkTestCase(test.TrialTestCase): address = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) address2 = self.service.allocate_fixed_ip(self.projects[1].id, - self.instance_id) + self.instance2_id) - net = service.get_network_for_project(self.projects[0].id) - net2 = service.get_network_for_project(self.projects[1].id) + net = db.project_get_network(None, self.projects[0].id) + net2 = db.project_get_network(None, self.projects[1].id) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) @@ -151,7 +153,7 @@ class NetworkTestCase(test.TrialTestCase): address = self.service.allocate_fixed_ip(project_id, self.instance_id) address2 = self.service.allocate_fixed_ip(project_id, self.instance_id) address3 = self.service.allocate_fixed_ip(project_id, self.instance_id) - net = service.get_network_for_project(project_id) + net = db.project_get_network(None, project_id) issue_ip(address, net.bridge) issue_ip(address2, net.bridge) issue_ip(address3, net.bridge) @@ -167,7 +169,7 @@ class NetworkTestCase(test.TrialTestCase): release_ip(address, net.bridge) release_ip(address2, net.bridge) release_ip(address3, net.bridge) - net = service.get_network_for_project(self.projects[0].id) + net = db.project_get_network(None, self.projects[0].id) self.service.deallocate_fixed_ip(first) def test_vpn_ip_and_port_looks_valid(self): @@ -186,7 +188,7 @@ class NetworkTestCase(test.TrialTestCase): self.service.set_network_host(project.id) projects.append(project) project = self.manager.create_project('boom' , self.user) - self.assertRaises(NoMoreNetworks, + self.assertRaises(db.NoMoreNetworks, self.service.set_network_host, project.id) self.manager.delete_project(project) @@ -198,7 +200,7 @@ class NetworkTestCase(test.TrialTestCase): """Makes sure that ip addresses that are deallocated get reused""" address = self.service.allocate_fixed_ip(self.projects[0].id, self.instance_id) - net = service.get_network_for_project(self.projects[0].id) + net = db.project_get_network(None, self.projects[0].id) issue_ip(address, net.bridge) self.service.deallocate_fixed_ip(address) release_ip(address, net.bridge) @@ -219,7 +221,7 @@ class NetworkTestCase(test.TrialTestCase): There are ips reserved at the bottom and top of the range. services (network, gateway, CloudPipe, broadcast) """ - network = service.get_network_for_project(self.projects[0].id) + network = db.project_get_network(None, self.projects[0].id) net_size = flags.FLAGS.network_size total_ips = (available_ips(network) + reserved_ips(network) + @@ -229,7 +231,7 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_addresses(self): """Test for a NoMoreAddresses exception when all fixed ips are used. """ - network = service.get_network_for_project(self.projects[0].id) + network = db.project_get_network(None, self.projects[0].id) # Number of availaible ips is len of the available list @@ -242,7 +244,7 @@ class NetworkTestCase(test.TrialTestCase): issue_ip(addresses[i],network.bridge) self.assertEqual(available_ips(network), 0) - self.assertRaises(NoMoreAddresses, + self.assertRaises(db.NoMoreAddresses, self.service.allocate_fixed_ip, self.projects[0].id, self.instance_id) @@ -274,11 +276,11 @@ def reserved_ips(network): def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" - fixed_ip = models.FixedIp.find_by_ip_str(address) - project_net = service.get_network_for_project(project_id) + fixed_ip = db.fixed_ip_get_by_address(None, address) + project_net = db.project_get_network(None, project_id) # instance exists until release - logging.error('fixed_ip.instance: %s', fixed_ip.instance) - logging.error('project_net: %s', project_net) + logging.debug('fixed_ip.instance: %s', fixed_ip.instance) + logging.debug('project_net: %s', project_net) return fixed_ip.instance is not None and fixed_ip.network == project_net From f1ab66bc1e6e1a6e0514820acd68d2feb29ec19a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 23 Aug 2010 20:39:19 -0700 Subject: [PATCH 036/113] fix daemons and move network code --- nova/auth/manager.py | 8 +++++--- nova/tests/network_unittest.py | 11 +++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index fc9aec07..e4d4afb7 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -524,8 +524,11 @@ class AuthManager(object): member_users) if project_dict: project = Project(**project_dict) - # FIXME(ja): EVIL HACK - db.network_create(context, {'project_id': project.id}) + try: + db.network_allocate(context, project.id) + except: + drv.delete_project(project.id) + raise return project def add_to_project(self, user, project): @@ -574,7 +577,6 @@ class AuthManager(object): def delete_project(self, project, context=None): """Deletes a project""" - # FIXME(ja): EVIL HACK network_ref = db.project_get_network(context, Project.safe_id(project)) try: diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 3552a77b..afa21767 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -179,18 +179,17 @@ class NetworkTestCase(test.TrialTestCase): FLAGS.num_networks) def test_too_many_networks(self): - """Ensure error is raised if we run out of vpn ports""" + """Ensure error is raised if we run out of networks""" projects = [] + # TODO(vish): use data layer for count networks_left = FLAGS.num_networks - models.Network.count() for i in range(networks_left): project = self.manager.create_project('many%s' % i, self.user) - self.service.set_network_host(project.id) projects.append(project) - project = self.manager.create_project('boom' , self.user) self.assertRaises(db.NoMoreNetworks, - self.service.set_network_host, - project.id) - self.manager.delete_project(project) + self.manager.create_project, + 'boom', + self.user) for project in projects: self.manager.delete_project(project) From 44aac6ff0474cfc417db62220e71ba34b6b86822 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 23 Aug 2010 22:06:49 -0700 Subject: [PATCH 037/113] moving network code and fixing run_instances --- nova/endpoint/cloud.py | 49 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index dd489cd9..db79c585 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -522,16 +522,16 @@ class CloudController(object): @defer.inlineCallbacks def _get_network_topic(self, context): """Retrieves the network host for a project""" - host = network_service.get_host_for_project(context.project.id) + network_ref = db.project_get_network(context, context.project.id) + host = db.network_get_host(context, network_ref['id']) if not host: host = yield rpc.call(FLAGS.network_topic, {"method": "set_network_host", - "args": {"user_id": context.user.id, - "project_id": context.project.id}}) - defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) + "args": {"project_id": context.project.id}}) + defer.returnValue(db.queue_get_for(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') - #@defer.inlineCallbacks + @defer.inlineCallbacks def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists @@ -571,11 +571,16 @@ class CloudController(object): for num in range(int(kwargs['max_count'])): inst = {} - inst['mac_address'] = utils.generate_mac() - inst['fixed_ip'] = db.fixed_ip_allocate_address(context, network_ref['id']) inst['image_id'] = image_id inst['kernel_id'] = kernel_id inst['ramdisk_id'] = ramdisk_id + instance_ref = db.instance_create(context, inst) + inst_id = instance_ref['id'] + if db.instance_is_vpn(instance_ref['id']): + fixed_ip = db.fixed_ip_allocate(context, network_ref['id']) + else: + fixed_ip = db.network_get_vpn_ip(context, network_ref['id']) + inst['mac_address'] = utils.generate_mac() inst['user_data'] = kwargs.get('user_data', '') inst['instance_type'] = kwargs.get('instance_type', 'm1.small') inst['reservation_id'] = reservation_id @@ -585,16 +590,23 @@ class CloudController(object): inst['project_id'] = context.project.id # FIXME(ja) inst['launch_index'] = num inst['security_group'] = security_group - # inst['hostname'] = inst.id # FIXME(ja): id isn't assigned until create + inst['hostname'] = inst_id # FIXME(ja): id isn't assigned until create + db.instance_update(context, inst_id, inst) + + + # TODO(vish): This probably should be done in the scheduler + # network is setup when host is assigned + network_topic = yield self.get_network_topic() + rpc.call(network_topic, + {"method": "setup_fixed_ip", + "args": {"fixed_ip": fixed_ip['id']}}) - inst_id = db.instance_create(context, inst) rpc.cast(FLAGS.compute_topic, {"method": "run_instance", "args": {"instance_id": inst_id}}) logging.debug("Casting to node for %s/%s's instance %s" % (context.project.name, context.user.name, inst_id)) - # defer.returnValue(self._format_instances(context, reservation_id)) - return self._format_run_instances(context, reservation_id) + defer.returnValue(self._format_instances(context, reservation_id)) @rbac.allow('projectmanager', 'sysadmin') @@ -605,13 +617,12 @@ class CloudController(object): for name in instance_id: logging.debug("Going to try and terminate %s" % name) try: - inst_id = name[2:] # remove the i- - instance_ref = db.instance_get(context, inst_id) + instance_ref = db.instance_get_by_name(context, name) except exception.NotFound: logging.warning("Instance %s was not found during terminate" % name) continue - + # FIXME(ja): where should network deallocate occur? # floating_ip = network_model.get_public_ip_for_instance(i) # if floating_ip: @@ -622,7 +633,7 @@ class CloudController(object): # rpc.cast(network_topic, # {"method": "disassociate_floating_ip", # "args": {"floating_ip": floating_ip}}) - # + # # fixed_ip = instance.get('private_dns_name', None) # if fixed_ip: # logging.debug("Deallocating address %s" % fixed_ip) @@ -633,14 +644,14 @@ class CloudController(object): # {"method": "deallocate_fixed_ip", # "args": {"fixed_ip": fixed_ip}}) - if instance_ref['physical_node_id'] is not None: + host = db.instance_get_host(context, instance_ref['id']) + if host is not None: # NOTE(joshua?): It's also internal default - rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, - instance_ref['physical_node_id']), + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "terminate_instance", "args": {"instance_id": name}}) else: - db.instance_destroy(context, inst_id) + db.instance_destroy(context, instance_ref['id']) # defer.returnValue(True) return True From 341cbd9c77b7e6d48319ce9e4f0a6b465ee5157c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 23 Aug 2010 22:25:09 -0700 Subject: [PATCH 038/113] bunch more fixes --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index db79c585..1c4c3483 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -599,7 +599,7 @@ class CloudController(object): network_topic = yield self.get_network_topic() rpc.call(network_topic, {"method": "setup_fixed_ip", - "args": {"fixed_ip": fixed_ip['id']}}) + "args": {"fixed_ip_id": fixed_ip['id']}}) rpc.cast(FLAGS.compute_topic, {"method": "run_instance", From 28aa68d4422b5154b1a6da2be9db57c4513f37fe Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 24 Aug 2010 01:30:48 -0700 Subject: [PATCH 039/113] typo in release_ip --- bin/nova-dhcpbridge | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index cd091739..018293e2 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -70,7 +70,7 @@ def del_lease(_mac, ip_address, _hostname, _interface): else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "release_fixed_ip", - "args": {"fixed_ip": ip_address}}) + "args": {"address": ip_address}}) def init_leases(interface): From 0fca373c5d614caac78314d85bbea0add3b9816c Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 15:23:52 -0400 Subject: [PATCH 040/113] getting run/terminate/describe to work --- nova/endpoint/cloud.py | 154 ++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 78 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index e64005c2..dd489cd9 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -29,7 +29,7 @@ import time from twisted.internet import defer -from nova import datastore +from nova import db from nova import exception from nova import flags from nova import models @@ -407,22 +407,22 @@ class CloudController(object): assert len(i) == 1 return i[0] - def _format_instances(self, context, reservation_id = None): + def _format_instances(self, context, reservation_id=None): reservations = {} - if context.user.is_admin(): - instgenerator = models.Instance.all() + if reservation_id: + instances = db.instance_get_by_reservation(context, reservation_id) else: - instgenerator = models.Instance.all() # FIXME - for instance in instgenerator: - res_id = instance.reservation_id - if reservation_id != None and reservation_id != res_id: - continue + if not context.user.is_admin(): + instances = db.instance_get_all(context) + else: + instances = db.instance_get_by_project(context, context.project.id) + for instance in instances: if not context.user.is_admin(): if instance['image_id'] == FLAGS.vpn_image_id: continue i = {} - i['instanceId'] = instance.name - i['imageId'] = instance.image_id + i['instanceId'] = instance['name'] + i['imageId'] = instance['image_id'] i['instanceState'] = { 'code': instance.state, 'name': instance.state_description @@ -442,14 +442,14 @@ class CloudController(object): i['instance_type'] = instance.instance_type i['launch_time'] = instance.created_at i['ami_launch_index'] = instance.launch_index - if not reservations.has_key(res_id): + if not reservations.has_key(instance['reservation_id']): r = {} - r['reservation_id'] = res_id + r['reservation_id'] = instance['reservation_id'] r['owner_id'] = instance.project_id r['group_set'] = self._convert_to_set([], 'groups') r['instances_set'] = [] - reservations[res_id] = r - reservations[res_id]['instances_set'].append(i) + reservations[instance['reservation_id']] = r + reservations[instance['reservation_id']]['instances_set'].append(i) return list(reservations.values()) @@ -563,88 +563,86 @@ class CloudController(object): raise exception.ApiError('Key Pair %s not found' % kwargs['key_name']) key_data = key_pair.public_key - # network_topic = yield self._get_network_topic(context) + # TODO: Get the real security group of launch in here security_group = "default" + + network_ref = db.project_get_network(context, context.project.id) + for num in range(int(kwargs['max_count'])): - is_vpn = False - if image_id == FLAGS.vpn_image_id: - is_vpn = True - inst = models.Instance() - #allocate_data = yield rpc.call(network_topic, - # {"method": "allocate_fixed_ip", - # "args": {"user_id": context.user.id, - # "project_id": context.project.id, - # "security_group": security_group, - # "is_vpn": is_vpn, - # "hostname": inst.instance_id}}) - allocate_data = {'mac_address': utils.generate_mac(), - 'fixed_ip': '192.168.0.100'} - inst.image_id = image_id - inst.kernel_id = kernel_id - inst.ramdisk_id = ramdisk_id - inst.user_data = kwargs.get('user_data', '') - inst.instance_type = kwargs.get('instance_type', 'm1.small') - inst.reservation_id = reservation_id - inst.key_data = key_data - inst.key_name = kwargs.get('key_name', None) - inst.user_id = context.user.id - inst.project_id = context.project.id - inst.launch_index = num - inst.security_group = security_group - inst.hostname = inst.id - for (key, value) in allocate_data.iteritems(): - setattr(inst, key, value) - inst.save() + inst = {} + inst['mac_address'] = utils.generate_mac() + inst['fixed_ip'] = db.fixed_ip_allocate_address(context, network_ref['id']) + inst['image_id'] = image_id + inst['kernel_id'] = kernel_id + inst['ramdisk_id'] = ramdisk_id + inst['user_data'] = kwargs.get('user_data', '') + inst['instance_type'] = kwargs.get('instance_type', 'm1.small') + inst['reservation_id'] = reservation_id + inst['key_data'] = key_data + inst['key_name'] = kwargs.get('key_name', None) + inst['user_id'] = context.user.id # FIXME(ja) + inst['project_id'] = context.project.id # FIXME(ja) + inst['launch_index'] = num + inst['security_group'] = security_group + # inst['hostname'] = inst.id # FIXME(ja): id isn't assigned until create + + inst_id = db.instance_create(context, inst) rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id": inst.id}}) - logging.debug("Casting to node for %s's instance with IP of %s" % - (context.user.name, inst.fixed_ip)) + "args": {"instance_id": inst_id}}) + logging.debug("Casting to node for %s/%s's instance %s" % + (context.project.name, context.user.name, inst_id)) # defer.returnValue(self._format_instances(context, reservation_id)) return self._format_run_instances(context, reservation_id) + @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks + # @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") - network_topic = yield self._get_network_topic(context) - for i in instance_id: - logging.debug("Going to try and terminate %s" % i) + # network_topic = yield self._get_network_topic(context) + for name in instance_id: + logging.debug("Going to try and terminate %s" % name) try: - instance = self._get_instance(context, i) + inst_id = name[2:] # remove the i- + instance_ref = db.instance_get(context, inst_id) except exception.NotFound: logging.warning("Instance %s was not found during terminate" - % i) + % name) continue - floating_ip = network_model.get_public_ip_for_instance(i) - if floating_ip: - logging.debug("Disassociating address %s" % floating_ip) - # NOTE(vish): Right now we don't really care if the ip is - # disassociated. We may need to worry about - # checking this later. Perhaps in the scheduler? - rpc.cast(network_topic, - {"method": "disassociate_floating_ip", - "args": {"floating_ip": floating_ip}}) + + # FIXME(ja): where should network deallocate occur? + # floating_ip = network_model.get_public_ip_for_instance(i) + # if floating_ip: + # logging.debug("Disassociating address %s" % floating_ip) + # # NOTE(vish): Right now we don't really care if the ip is + # # disassociated. We may need to worry about + # # checking this later. Perhaps in the scheduler? + # rpc.cast(network_topic, + # {"method": "disassociate_floating_ip", + # "args": {"floating_ip": floating_ip}}) + # + # fixed_ip = instance.get('private_dns_name', None) + # if fixed_ip: + # logging.debug("Deallocating address %s" % fixed_ip) + # # NOTE(vish): Right now we don't really care if the ip is + # # actually removed. We may need to worry about + # # checking this later. Perhaps in the scheduler? + # rpc.cast(network_topic, + # {"method": "deallocate_fixed_ip", + # "args": {"fixed_ip": fixed_ip}}) - fixed_ip = instance.get('private_dns_name', None) - if fixed_ip: - logging.debug("Deallocating address %s" % fixed_ip) - # NOTE(vish): Right now we don't really care if the ip is - # actually removed. We may need to worry about - # checking this later. Perhaps in the scheduler? - rpc.cast(network_topic, - {"method": "deallocate_fixed_ip", - "args": {"fixed_ip": fixed_ip}}) - - if instance.get('node_name', 'unassigned') != 'unassigned': + if instance_ref['physical_node_id'] is not None: # NOTE(joshua?): It's also internal default - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, + instance_ref['physical_node_id']), {"method": "terminate_instance", - "args": {"instance_id": i}}) + "args": {"instance_id": name}}) else: - instance.destroy() - defer.returnValue(True) + db.instance_destroy(context, inst_id) + # defer.returnValue(True) + return True @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): From 0ef338e2711fe423e3edd4ed931c544955377bc0 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 18:01:32 -0400 Subject: [PATCH 041/113] more cleanup --- nova/endpoint/cloud.py | 81 +++++++++++++---------------------------- nova/endpoint/images.py | 10 ++++- 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index dd489cd9..afb62cc6 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -64,13 +64,12 @@ class CloudController(object): sent to the other nodes. """ def __init__(self): - self.instdir = model.InstanceDirectory() self.setup() @property def instances(self): """ All instances in the system, as dicts """ - return self.instdir.all + return db.instance_get_all(None) @property def volumes(self): @@ -84,6 +83,8 @@ class CloudController(object): def setup(self): """ Ensure the keychains and folders exist. """ + # FIXME(ja): this should be moved to a nova-manage command, + # if not setup throw exceptions instead of running # Create keys folder, if it doesn't exist if not os.path.exists(FLAGS.keys_path): os.makedirs(FLAGS.keys_path) @@ -92,27 +93,23 @@ class CloudController(object): if not os.path.exists(root_ca_path): start = os.getcwd() os.chdir(FLAGS.ca_path) + # TODO: Do this with M2Crypto instead utils.runthis("Generating root CA: %s", "sh genrootca.sh") os.chdir(start) - # TODO: Do this with M2Crypto instead - - def get_instance_by_ip(self, ip): - return self.instdir.by_ip(ip) def _get_mpi_data(self, project_id): result = {} - for instance in self.instdir.all: - if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], - INSTANCE_TYPES[instance['instance_type']]['vcpus']) - if instance['key_name'] in result: - result[instance['key_name']].append(line) - else: - result[instance['key_name']] = [line] + for instance in db.instance_get_by_project(project_id): + line = '%s slots=%d' % (instance['private_dns_name'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) + if instance['key_name'] in result: + result[instance['key_name']].append(line) + else: + result[instance['key_name']] = [line] return result def get_metadata(self, ipaddress): - i = self.get_instance_by_ip(ipaddress) + i = db.instance_get_by_ip(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -252,17 +249,11 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def get_console_output(self, context, instance_id, **kwargs): # instance_id is passed in as a list of instances - instance = self._get_instance(context, instance_id[0]) + instance = db.instance_get(context, instance_id[0]) return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "get_console_output", "args": {"instance_id": instance_id[0]}}) - def _get_user_id(self, context): - if context and context.user: - return context.user.id - else: - return None - @rbac.allow('projectmanager', 'sysadmin') def describe_volumes(self, context, **kwargs): volumes = [] @@ -301,12 +292,12 @@ class CloudController(object): @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it - result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", + volume_id = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", "args": {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary - volume = self._get_volume(context, result) + volume = db.volume_get(context, volume_id) defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) def _get_address(self, context, public_ip): @@ -316,31 +307,9 @@ class CloudController(object): return address raise exception.NotFound("Address at ip %s not found" % public_ip) - def _get_image(self, context, image_id): - """passes in context because - objectstore does its own authorization""" - result = images.list(context, [image_id]) - if not result: - raise exception.NotFound('Image %s could not be found' % image_id) - image = result[0] - return image - - def _get_instance(self, context, instance_id): - for instance in self.instdir.all: - if instance['instance_id'] == instance_id: - if context.user.is_admin() or instance['project_id'] == context.project.id: - return instance - raise exception.NotFound('Instance %s could not be found' % instance_id) - - def _get_volume(self, context, volume_id): - volume = service.get_volume(volume_id) - if context.user.is_admin() or volume['project_id'] == context.project.id: - return volume - raise exception.NotFound('Volume %s could not be found' % volume_id) - @rbac.allow('projectmanager', 'sysadmin') def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume = self._get_volume(context, volume_id) + volume = db.volume_get(context, volume_id) if volume['status'] == "attached": raise exception.ApiError("Volume is already attached") # TODO(vish): looping through all volumes is slow. We should probably maintain an index @@ -348,7 +317,7 @@ class CloudController(object): if vol['instance_id'] == instance_id and vol['mountpoint'] == device: raise exception.ApiError("Volume %s is already attached to %s" % (vol['volume_id'], vol['mountpoint'])) volume.start_attach(instance_id, device) - instance = self._get_instance(context, instance_id) + instance = db.instance_get(context, instance_id) compute_node = instance['node_name'] rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), {"method": "attach_volume", @@ -364,7 +333,7 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): - volume = self._get_volume(context, volume_id) + volume = db.volume_get(context, volume_id) instance_id = volume.get('instance_id', None) if not instance_id: raise exception.Error("Volume isn't attached to anything!") @@ -372,7 +341,7 @@ class CloudController(object): raise exception.Error("Volume is already detached") try: volume.start_detach() - instance = self._get_instance(context, instance_id) + instance = db.instance_get(context, instance_id) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "detach_volume", "args": {"instance_id": instance_id, @@ -499,7 +468,7 @@ class CloudController(object): @rbac.allow('netadmin') @defer.inlineCallbacks def associate_address(self, context, instance_id, public_ip, **kwargs): - instance = self._get_instance(context, instance_id) + instance = db.instance_get(context, instance_id) address = self._get_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, @@ -536,7 +505,7 @@ class CloudController(object): # make sure user can access the image # vpn image is private so it doesn't show up on lists if kwargs['image_id'] != FLAGS.vpn_image_id: - image = self._get_image(context, kwargs['image_id']) + image = images.get(context, kwargs['image_id']) # FIXME(ja): if image is cloudpipe, this breaks @@ -550,8 +519,8 @@ class CloudController(object): ramdisk_id = kwargs.get('ramdisk_id', ramdisk_id) # make sure we have access to kernel and ramdisk - self._get_image(context, kernel_id) - self._get_image(context, ramdisk_id) + images.get(context, kernel_id) + images.get(context, ramdisk_id) logging.debug("Going to run instances...") reservation_id = utils.generate_uid('r') @@ -648,7 +617,7 @@ class CloudController(object): def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" for i in instance_id: - instance = self._get_instance(context, i) + instance = db.instance_get(context, i) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "reboot_instance", "args": {"instance_id": i}}) @@ -657,7 +626,7 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized - volume = self._get_volume(context, volume_id) + volume = db.volume_get(context, volume_id) volume_node = volume['node_name'] rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), {"method": "delete_volume", diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index 2a88d66a..f72c277a 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -26,6 +26,7 @@ import urllib import boto.s3.connection +from nova import exception from nova import flags from nova import utils from nova.auth import manager @@ -55,7 +56,6 @@ def register(context, image_location): return image_id - def list(context, filter_list=[]): """ return a list of all images that a user can see @@ -71,6 +71,14 @@ def list(context, filter_list=[]): return [i for i in result if i['imageId'] in filter_list] return result +def get(context, image_id): + """return a image object if the context has permissions""" + result = list(context, [image_id]) + if not result: + raise exception.NotFound('Image %s could not be found' % image_id) + image = result[0] + return image + def deregister(context, image_id): """ unregister an image """ From 6451c4cf69923447f7b4929a3dd3403871ee3eb2 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 18:30:06 -0400 Subject: [PATCH 042/113] more work on getting running instances to work --- nova/endpoint/cloud.py | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 826a4cb4..6f937022 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -504,11 +504,12 @@ class CloudController(object): def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists - if kwargs['image_id'] != FLAGS.vpn_image_id: + vpn = kwargs['image_id'] == FLAGS.vpn_image_id + + if not vpn: image = images.get(context, kwargs['image_id']) - # FIXME(ja): if image is cloudpipe, this breaks - + # FIXME(ja): if image is vpn, this breaks # get defaults from imagestore image_id = image['imageId'] kernel_id = image.get('kernelId', FLAGS.default_kernel) @@ -523,7 +524,6 @@ class CloudController(object): images.get(context, ramdisk_id) logging.debug("Going to run instances...") - reservation_id = utils.generate_uid('r') launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) key_data = None if kwargs.has_key('key_name'): @@ -537,35 +537,38 @@ class CloudController(object): security_group = "default" network_ref = db.project_get_network(context, context.project.id) + + base_options = {} + base_options['image_id'] = image_id + base_options['kernel_id'] = kernel_id + base_options['ramdisk_id'] = ramdisk_id + base_options['reservation_id'] = utils.generate_uid('r') + base_options['key_data'] = key_data + base_options['key_name'] = kwargs.get('key_name', None) + base_options['user_id'] = context.user.id + base_options['project_id'] = context.project.id + base_options['user_data'] = kwargs.get('user_data', '') + base_options['instance_type'] = kwargs.get('instance_type', 'm1.small') + base_options['security_group'] = security_group for num in range(int(kwargs['max_count'])): - inst = {} - inst['image_id'] = image_id - inst['kernel_id'] = kernel_id - inst['ramdisk_id'] = ramdisk_id - instance_ref = db.instance_create(context, inst) - inst_id = instance_ref['id'] - if db.instance_is_vpn(instance_ref['id']): - fixed_ip = db.fixed_ip_allocate(context, network_ref['id']) - else: + inst_id = db.instance_create(context, base_options) + + if vpn: fixed_ip = db.network_get_vpn_ip(context, network_ref['id']) + else: + fixed_ip = db.fixed_ip_allocate(context, network_ref['id']) + + inst = {} inst['mac_address'] = utils.generate_mac() - inst['user_data'] = kwargs.get('user_data', '') - inst['instance_type'] = kwargs.get('instance_type', 'm1.small') - inst['reservation_id'] = reservation_id - inst['key_data'] = key_data - inst['key_name'] = kwargs.get('key_name', None) - inst['user_id'] = context.user.id # FIXME(ja) - inst['project_id'] = context.project.id # FIXME(ja) inst['launch_index'] = num - inst['security_group'] = security_group - inst['hostname'] = inst_id # FIXME(ja): id isn't assigned until create + inst['hostname'] = inst_id db.instance_update(context, inst_id, inst) # TODO(vish): This probably should be done in the scheduler # network is setup when host is assigned - network_topic = yield self.get_network_topic() + network_topic = yield self._get_network_topic(context) rpc.call(network_topic, {"method": "setup_fixed_ip", "args": {"fixed_ip": fixed_ip['id']}}) From 9a0ea6b9b2ff71f3f1e48e0f10babc892378b562 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 22:41:34 -0400 Subject: [PATCH 043/113] work towards volumes using db layer --- nova/endpoint/cloud.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 97d978cc..e261abc7 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -256,11 +256,13 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def describe_volumes(self, context, **kwargs): - volumes = [] - for volume in self.volumes: - if context.user.is_admin() or volume['project_id'] == context.project.id: - v = self.format_volume(context, volume) - volumes.append(v) + if context.user.is_admin(): + volumes = db.volume_get_all(context) + else: + volumes = db.volume_get_by_project(context, context.project.id) + + voluems = [self.format_volume(context, v) for v in volumes] + return defer.succeed({'volumeSet': volumes}) def format_volume(self, context, volume): From 2a24aba7d8e746119734da68718e6ec72b00dd7f Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 22:51:48 -0400 Subject: [PATCH 044/113] move create volume to work like instances --- nova/endpoint/cloud.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index e261abc7..e7e751b5 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -261,24 +261,24 @@ class CloudController(object): else: volumes = db.volume_get_by_project(context, context.project.id) - voluems = [self.format_volume(context, v) for v in volumes] + volumes = [self.format_volume(context, v) for v in volumes] - return defer.succeed({'volumeSet': volumes}) + return {'volumeSet': volumes} def format_volume(self, context, volume): v = {} - v['volumeId'] = volume['volume_id'] + v['volumeId'] = volume['id'] v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] - v['createTime'] = volume['create_time'] + # v['createTime'] = volume['create_time'] if context.user.is_admin(): v['status'] = '%s (%s, %s, %s, %s)' % ( - volume.get('status', None), - volume.get('user_id', None), - volume.get('node_name', None), - volume.get('instance_id', ''), - volume.get('mountpoint', '')) + volume['status'], + volume['user_id'], + 'node_name', + volume['instance_id'], + volume['mountpoint']) if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], 'deleteOnTermination': volume['delete_on_termination'], @@ -293,11 +293,18 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): - # TODO(vish): refactor this to create the volume object here and tell service to create it - volume_id = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args": {"size": size, - "user_id": context.user.id, - "project_id": context.project.id}}) + vol = {} + vol['size'] = size + vol['user_id'] = context.user.id + vol['project_id'] = context.project.id + vol['availability_zone'] = FLAGS.storage_availability_zone + vol['status'] = "creating" + vol['attach_status'] = "detached" + volume_id = db.volume_create(context, vol) + + yield rpc.cast(FLAGS.volume_topic, {"method": "create_volume", + "args": {"volume_id": volume_id}}) + # NOTE(vish): rpc returned value is in the result key in the dictionary volume = db.volume_get(context, volume_id) defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) From ed935eb57683ed00b60e5b5831d86e7c72c98285 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 24 Aug 2010 23:09:00 -0400 Subject: [PATCH 045/113] small tweaks --- nova/endpoint/cloud.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index e7e751b5..64a705e6 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -261,11 +261,11 @@ class CloudController(object): else: volumes = db.volume_get_by_project(context, context.project.id) - volumes = [self.format_volume(context, v) for v in volumes] + volumes = [self._format_volume(context, v) for v in volumes] return {'volumeSet': volumes} - def format_volume(self, context, volume): + def _format_volume(self, context, volume): v = {} v['volumeId'] = volume['id'] v['status'] = volume['status'] @@ -305,9 +305,8 @@ class CloudController(object): yield rpc.cast(FLAGS.volume_topic, {"method": "create_volume", "args": {"volume_id": volume_id}}) - # NOTE(vish): rpc returned value is in the result key in the dictionary volume = db.volume_get(context, volume_id) - defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) + defer.returnValue({'volumeSet': [self._format_volume(context, volume)]}) def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py @@ -343,8 +342,7 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): volume = db.volume_get(context, volume_id) - instance_id = volume.get('instance_id', None) - if not instance_id: + if volume['instance_id'] is None: raise exception.Error("Volume isn't attached to anything!") if volume['status'] == "available": raise exception.Error("Volume is already detached") From f2f59482375afec43b3e1065b04228baa64f7ef5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 25 Aug 2010 13:14:49 -0700 Subject: [PATCH 046/113] more data layer breakouts, lots of fixes to cloud.py --- nova/endpoint/cloud.py | 221 ++++++++++++++------------------- nova/models.py | 37 ++++-- nova/tests/network_unittest.py | 2 +- 3 files changed, 121 insertions(+), 139 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 64a705e6..ffe3d3cc 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -32,16 +32,12 @@ from twisted.internet import defer from nova import db from nova import exception from nova import flags -from nova import models from nova import rpc from nova import utils from nova.auth import rbac from nova.auth import manager -from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES from nova.endpoint import images -from nova.network import service as network_service -from nova.volume import service FLAGS = flags.FLAGS @@ -66,18 +62,6 @@ class CloudController(object): def __init__(self): self.setup() - @property - def instances(self): - """ All instances in the system, as dicts """ - return db.instance_get_all(None) - - @property - def volumes(self): - """ returns a list of all volumes """ - for volume_id in datastore.Redis.instance().smembers("volumes"): - volume = service.get_volume(volume_id) - yield volume - def __str__(self): return 'CloudController' @@ -100,7 +84,7 @@ class CloudController(object): def _get_mpi_data(self, project_id): result = {} for instance in db.instance_get_by_project(project_id): - line = '%s slots=%d' % (instance['private_dns_name'], + line = '%s slots=%d' % (instance.fixed_ip['str_id'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) @@ -109,7 +93,7 @@ class CloudController(object): return result def get_metadata(self, ipaddress): - i = db.instance_get_by_ip(ipaddress) + i = db.instance_get_by_address(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -122,12 +106,7 @@ class CloudController(object): } else: keys = '' - - address_record = network_model.FixedIp(i['private_dns_name']) - if address_record: - hostname = address_record['hostname'] - else: - hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-') + hostname = i['hostname'] data = { 'user-data': base64.b64decode(i['user_data']), 'meta-data': { @@ -249,10 +228,11 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def get_console_output(self, context, instance_id, **kwargs): # instance_id is passed in as a list of instances - instance = db.instance_get(context, instance_id[0]) - return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "get_console_output", - "args": {"instance_id": instance_id[0]}}) + instance_ref = db.instance_get_by_str(context, instance_id[0]) + return rpc.call('%s.%s' % (FLAGS.compute_topic, + instance_ref['node_name']), + {"method": "get_console_output", + "args": {"instance_id": instance_ref['id']}}) @rbac.allow('projectmanager', 'sysadmin') def describe_volumes(self, context, **kwargs): @@ -267,7 +247,7 @@ class CloudController(object): def _format_volume(self, context, volume): v = {} - v['volumeId'] = volume['id'] + v['volumeId'] = volume['str_id'] v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] @@ -298,7 +278,7 @@ class CloudController(object): vol['user_id'] = context.user.id vol['project_id'] = context.project.id vol['availability_zone'] = FLAGS.storage_availability_zone - vol['status'] = "creating" + vol['status'] = "creating" vol['attach_status'] = "detached" volume_id = db.volume_create(context, vol) @@ -308,61 +288,54 @@ class CloudController(object): volume = db.volume_get(context, volume_id) defer.returnValue({'volumeSet': [self._format_volume(context, volume)]}) - def _get_address(self, context, public_ip): - # FIXME(vish) this should move into network.py - address = network_model.FloatingIp.lookup(public_ip) - if address and (context.user.is_admin() or address['project_id'] == context.project.id): - return address - raise exception.NotFound("Address at ip %s not found" % public_ip) @rbac.allow('projectmanager', 'sysadmin') def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume = db.volume_get(context, volume_id) - if volume['status'] == "attached": + volume_ref = db.volume_get_by_str(context, volume_id) + # TODO(vish): abstract status checking? + if volume_ref['status'] == "attached": raise exception.ApiError("Volume is already attached") - # TODO(vish): looping through all volumes is slow. We should probably maintain an index - for vol in self.volumes: - if vol['instance_id'] == instance_id and vol['mountpoint'] == device: - raise exception.ApiError("Volume %s is already attached to %s" % (vol['volume_id'], vol['mountpoint'])) - volume.start_attach(instance_id, device) - instance = db.instance_get(context, instance_id) - compute_node = instance['node_name'] - rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), + #volume.start_attach(instance_id, device) + instance_ref = db.instance_get_by_str(context, instance_id) + host = db.instance_get_host(context, instance_ref['id']) + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "attach_volume", - "args": {"volume_id": volume_id, - "instance_id": instance_id, + "args": {"volume_id": volume_ref['id'], + "instance_id": instance_ref['id'], "mountpoint": device}}) - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, + return defer.succeed({'attachTime': volume_ref['attach_time'], + 'device': volume_ref['mountpoint'], + 'instanceId': instance_ref['id_str'], 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) + 'status': volume_ref['attach_status'], + 'volumeId': volume_ref['id']}) @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): - volume = db.volume_get(context, volume_id) - if volume['instance_id'] is None: + volume_ref = db.volume_get_by_str(context, volume_id) + instance_ref = db.volume_get_instance(context, volume_ref['id']) + if not instance_ref: raise exception.Error("Volume isn't attached to anything!") - if volume['status'] == "available": + # TODO(vish): abstract status checking? + if volume_ref['status'] == "available": raise exception.Error("Volume is already detached") try: - volume.start_detach() - instance = db.instance_get(context, instance_id) - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + #volume.start_detach() + host = db.instance_get_host(context, instance_ref['id']) + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "detach_volume", - "args": {"instance_id": instance_id, - "volume_id": volume_id}}) + "args": {"instance_id": instance_ref['id'], + "volume_id": volume_ref['id']}}) except exception.NotFound: # If the instance doesn't exist anymore, # then we need to call detach blind - volume.finish_detach() - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, + db.volume_detached(context) + return defer.succeed({'attachTime': volume_ref['attach_time'], + 'device': volume_ref['mountpoint'], + 'instanceId': instance_ref['id_str'], 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) + 'status': volume_ref['attach_status'], + 'volumeId': volume_ref['id']}) def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -397,15 +370,18 @@ class CloudController(object): if instance['image_id'] == FLAGS.vpn_image_id: continue i = {} - i['instanceId'] = instance['name'] + i['instanceId'] = instance['str_id'] i['imageId'] = instance['image_id'] i['instanceState'] = { - 'code': instance.state, - 'name': instance.state_description + 'code': instance['state'], + 'name': instance['state_description'] } - i['public_dns_name'] = None #network_model.get_public_ip_for_instance( - # i['instance_id']) - i['private_dns_name'] = instance.fixed_ip['ip_str'] + floating_addr = db.instance_get_floating_address(context, + instance['id']) + i['public_dns_name'] = floating_addr + fixed_addr = db.instance_get_fixed_address(context, + instance['id']) + i['private_dns_name'] = fixed_addr if not i['public_dns_name']: i['public_dns_name'] = i['private_dns_name'] i['dns_name'] = None @@ -435,20 +411,23 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in network_model.FloatingIp.all(): - # TODO(vish): implement a by_project iterator for addresses - if (context.user.is_admin() or - address['project_id'] == context.project.id): - address_rv = { - 'public_ip': address['address'], - 'instance_id': address.get('instance_id', 'free') - } - if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s, %s)" % ( - address['instance_id'], - address['user_id'], - address['project_id'], - ) + if context.user.is_admin(): + iterator = db.floating_ip_get_all(context) + else: + iterator = db.floating_ip_get_by_project(context, + context.project.id) + for floating_ip_ref in iterator: + address = floating_ip_ref['id_str'] + instance_ref = db.instance_get_by_address(address) + address_rv = { + 'public_ip': address, + 'instance_id': instance_ref['id_str'] + } + if context.user.is_admin(): + address_rv['instance_id'] = "%s (%s)" % ( + address_rv['instance_id'], + floating_ip_ref['project_id'], + ) addresses.append(address_rv) return {'addressesSet': addresses} @@ -458,41 +437,42 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) public_ip = yield rpc.call(network_topic, {"method": "allocate_floating_ip", - "args": {"user_id": context.user.id, - "project_id": context.project.id}}) + "args": {"project_id": context.project.id}}) defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @defer.inlineCallbacks def release_address(self, context, public_ip, **kwargs): # NOTE(vish): Should we make sure this works? + floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "deallocate_floating_ip", - "args": {"floating_ip": public_ip}}) + "args": {"floating_ip": floating_ip_ref['str_id']}}) defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') @defer.inlineCallbacks def associate_address(self, context, instance_id, public_ip, **kwargs): - instance = db.instance_get(context, instance_id) - address = self._get_address(context, public_ip) + instance_ref = db.instance_get_by_str(context, instance_id) + fixed_ip_ref = db.fixed_ip_get_by_instance(context, instance_ref['id']) + floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "associate_floating_ip", - "args": {"floating_ip": address['address'], - "fixed_ip": instance['private_dns_name'], - "instance_id": instance['instance_id']}}) + "args": {"floating_ip": floating_ip_ref['str_id'], + "fixed_ip": fixed_ip_ref['str_id'], + "instance_id": instance_ref['id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') @defer.inlineCallbacks def disassociate_address(self, context, public_ip, **kwargs): - address = self._get_address(context, public_ip) + floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "disassociate_floating_ip", - "args": {"floating_ip": address['address']}}) + "args": {"floating_ip": floating_ip_ref['str_id']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks @@ -596,13 +576,13 @@ class CloudController(object): def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") # network_topic = yield self._get_network_topic(context) - for name in instance_id: - logging.debug("Going to try and terminate %s" % name) + for id_str in instance_id: + logging.debug("Going to try and terminate %s" % id_str) try: - instance_ref = db.instance_get_by_name(context, name) + instance_ref = db.instance_get_by_str(context, id_str) except exception.NotFound: logging.warning("Instance %s was not found during terminate" - % name) + % id_str) continue # FIXME(ja): where should network deallocate occur? @@ -631,7 +611,7 @@ class CloudController(object): # NOTE(joshua?): It's also internal default rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "terminate_instance", - "args": {"instance_id": name}}) + "args": {"instance_id": instance_ref['id']}}) else: db.instance_destroy(context, instance_ref['id']) # defer.returnValue(True) @@ -640,19 +620,20 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" - for i in instance_id: - instance = db.instance_get(context, i) - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "reboot_instance", - "args": {"instance_id": i}}) + for id_str in instance_id: + instance_ref = db.instance_get_by_str(context, id_str) + host = db.instance_get_host(context, instance_ref['id']) + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "reboot_instance", + "args": {"instance_id": instance_ref['id']}}) return defer.succeed(True) @rbac.allow('projectmanager', 'sysadmin') def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized - volume = db.volume_get(context, volume_id) - volume_node = volume['node_name'] - rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), + volume_ref = db.volume_get_by_str(context, volume_id) + host = db.volume_get_host(context, volume_ref['id']) + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "delete_volume", "args": {"volume_id": volume_id}}) return defer.succeed(True) @@ -705,23 +686,3 @@ class CloudController(object): raise exception.ApiError('operation_type must be add or remove') result = images.modify(context, image_id, operation_type) return defer.succeed(result) - - def update_state(self, topic, value): - """ accepts status reports from the queue and consolidates them """ - # TODO(jmc): if an instance has disappeared from - # the node, call instance_death - if topic == "instances": - return defer.succeed(True) - aggregate_state = getattr(self, topic) - node_name = value.keys()[0] - items = value[node_name] - - logging.debug("Updating %s state for %s" % (topic, node_name)) - - for item_id in items.keys(): - if (aggregate_state.has_key('pending') and - aggregate_state['pending'].has_key(item_id)): - del aggregate_state['pending'][item_id] - aggregate_state[node_name] = items - - return defer.succeed(True) diff --git a/nova/models.py b/nova/models.py index c7ca9bb7..7ad37981 100644 --- a/nova/models.py +++ b/nova/models.py @@ -40,6 +40,7 @@ flags.DEFINE_string('sql_connection', class NovaBase(object): __table_args__ = {'mysql_engine':'InnoDB'} + __prefix__ = 'none' created_at = Column(DateTime) updated_at = Column(DateTime) @@ -86,6 +87,15 @@ class NovaBase(object): except exc.NoResultFound: raise exception.NotFound("No model for id %s" % obj_id) + @classmethod + def find_by_str(cls, str_id): + id = int(str_id.rpartition('-')[2]) + return cls.find(id) + + @property + def str_id(self): + return "%s-%s" % (self.__prefix__, self.id) + def save(self): session = NovaBase.get_session() session.add(self) @@ -109,6 +119,7 @@ class NovaBase(object): class Image(Base, NovaBase): __tablename__ = 'images' + __prefix__ = 'ami' id = Column(Integer, primary_key=True) user_id = Column(String(255))#, ForeignKey('users.id'), nullable=False) project_id = Column(String(255))#, ForeignKey('projects.id'), nullable=False) @@ -166,6 +177,7 @@ class Daemon(Base, NovaBase): class Instance(Base, NovaBase): __tablename__ = 'instances' + __prefix__ = 'i' id = Column(Integer, primary_key=True) user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) @@ -182,7 +194,7 @@ class Instance(Base, NovaBase): # TODO(vish): make this opaque somehow @property def name(self): - return "i-%s" % self.id + return self.str_id image_id = Column(Integer, ForeignKey('images.id'), nullable=True) @@ -198,7 +210,7 @@ class Instance(Base, NovaBase): state_description = Column(String(255)) hostname = Column(String(255)) - physical_node_id = Column(Integer) + node_name = Column(String(255)) #, ForeignKey('physical_node.id')) instance_type = Column(Integer) @@ -230,6 +242,7 @@ class Instance(Base, NovaBase): class Volume(Base, NovaBase): __tablename__ = 'volumes' + __prefix__ = 'vol' id = Column(Integer, primary_key=True) user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) @@ -267,15 +280,19 @@ class FixedIp(Base, NovaBase): leased = Column(Boolean, default=False) reserved = Column(Boolean, default=False) + @property + def str_id(self): + return self.ip_str + @classmethod - def find_by_ip_str(cls, ip_str): + def find_by_str(cls, str_id): session = NovaBase.get_session() try: - result = session.query(cls).filter_by(ip_str=ip_str).one() + result = session.query(cls).filter_by(ip_str=str_id).one() session.commit() return result except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % ip_str) + raise exception.NotFound("No model for ip str %s" % str_id) class FloatingIp(Base, NovaBase): @@ -288,15 +305,19 @@ class FloatingIp(Base, NovaBase): project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) node_name = Column(String(255)) #, ForeignKey('physical_node.id')) + @property + def str_id(self): + return self.ip_str + @classmethod - def find_by_ip_str(cls, ip_str): + def find_by_str(cls, str_id): session = NovaBase.get_session() try: - result = session.query(cls).filter_by(ip_str=ip_str).one() + result = session.query(cls).filter_by(ip_str=str_id).one() session.commit() return result except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % ip_str) + raise exception.NotFound("No model for ip str %s" % str_id) class Network(Base, NovaBase): diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index afa21767..d8a398aa 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -81,7 +81,7 @@ class NetworkTestCase(test.TrialTestCase): pubnet = IPy.IP(flags.FLAGS.public_range) ip_str = str(pubnet[0]) try: - floating_ip = models.FloatingIp.find_by_ip_str(ip_str) + floating_ip = models.FloatingIp.find_by_str(ip_str) except exception.NotFound: floating_ip = models.FloatingIp() floating_ip.ip_str = ip_str From c99aad8d2761b5feb5836a806c8e6610a62455f2 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 25 Aug 2010 16:37:22 -0500 Subject: [PATCH 047/113] Moved API tests into a sub-folder of the tests/ and added a stubbed-out test declarations to mirror existing API tickets --- nova/wsgi_test.py | 96 ----------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 nova/wsgi_test.py diff --git a/nova/wsgi_test.py b/nova/wsgi_test.py deleted file mode 100644 index 786dc1bc..00000000 --- a/nova/wsgi_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Test WSGI basics and provide some helper functions for other WSGI tests. -""" - -import unittest - -import routes -import webob - -from nova import wsgi - - -class Test(unittest.TestCase): - - def test_debug(self): - - class Application(wsgi.Application): - """Dummy application to test debug.""" - - def __call__(self, environ, start_response): - start_response("200", [("X-Test", "checking")]) - return ['Test result'] - - application = wsgi.Debug(Application()) - result = webob.Request.blank('/').get_response(application) - self.assertEqual(result.body, "Test result") - - def test_router(self): - - class Application(wsgi.Application): - """Test application to call from router.""" - - def __call__(self, environ, start_response): - start_response("200", []) - return ['Router result'] - - class Router(wsgi.Router): - """Test router.""" - - def __init__(self): - mapper = routes.Mapper() - mapper.connect("/test", controller=Application()) - super(Router, self).__init__(mapper) - - result = webob.Request.blank('/test').get_response(Router()) - self.assertEqual(result.body, "Router result") - result = webob.Request.blank('/bad').get_response(Router()) - self.assertNotEqual(result.body, "Router result") - - def test_controller(self): - - class Controller(wsgi.Controller): - """Test controller to call from router.""" - test = self - - def show(self, req, id): # pylint: disable-msg=W0622,C0103 - """Default action called for requests with an ID.""" - self.test.assertEqual(req.path_info, '/tests/123') - self.test.assertEqual(id, '123') - return id - - class Router(wsgi.Router): - """Test router.""" - - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=Controller()) - super(Router, self).__init__(mapper) - - result = webob.Request.blank('/tests/123').get_response(Router()) - self.assertEqual(result.body, "123") - result = webob.Request.blank('/test/123').get_response(Router()) - self.assertNotEqual(result.body, "123") - - def test_serializer(self): - # TODO(eday): Placeholder for serializer testing. - pass From f24b17391c37b33d83ddfe6572da21039b735443 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 25 Aug 2010 16:44:10 -0700 Subject: [PATCH 048/113] network tests pass again --- bin/nova-dhcpbridge | 4 - nova/flags.py | 19 +++-- nova/models.py | 5 -- nova/tests/network_unittest.py | 135 +++++++++++++++++---------------- 4 files changed, 80 insertions(+), 83 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 018293e2..6747a3a0 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -46,10 +46,6 @@ def add_lease(_mac, ip_address, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: logging.debug("leasing ip") - from nova import models - print models.FixedIp.count() - print models.Network.count() - print FLAGS.sql_connection service.VlanNetworkService().lease_fixed_ip(ip_address) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), diff --git a/nova/flags.py b/nova/flags.py index e3feb252..d4b2b7c3 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -22,6 +22,7 @@ where they're used. """ import getopt +import os import socket import sys @@ -34,7 +35,7 @@ class FlagValues(gflags.FlagValues): Unknown flags will be ignored when parsing the command line, but the command line will be kept so that it can be replayed if new flags are defined after the initial parsing. - + """ def __init__(self): @@ -50,7 +51,7 @@ class FlagValues(gflags.FlagValues): # leftover args at the end sneaky_unparsed_args = {"value": None} original_argv = list(argv) - + if self.IsGnuGetOpt(): orig_getopt = getattr(getopt, 'gnu_getopt') orig_name = 'gnu_getopt' @@ -74,14 +75,14 @@ class FlagValues(gflags.FlagValues): unparsed_args = sneaky_unparsed_args['value'] if unparsed_args: if self.IsGnuGetOpt(): - args = argv[:1] + unparsed + args = argv[:1] + unparsed_args else: args = argv[:1] + original_argv[-len(unparsed_args):] else: args = argv[:1] finally: setattr(getopt, orig_name, orig_getopt) - + # Store the arguments for later, we'll need them for new flags # added at runtime self.__dict__['__stored_argv'] = original_argv @@ -92,7 +93,7 @@ class FlagValues(gflags.FlagValues): def SetDirty(self, name): """Mark a flag as dirty so that accessing it will case a reparse.""" self.__dict__['__dirty'].append(name) - + def IsDirty(self, name): return name in self.__dict__['__dirty'] @@ -113,12 +114,12 @@ class FlagValues(gflags.FlagValues): for k in self.__dict__['__dirty']: setattr(self, k, getattr(new_flags, k)) self.ClearDirty() - + def __setitem__(self, name, flag): gflags.FlagValues.__setitem__(self, name, flag) if self.WasAlreadyParsed(): self.SetDirty(name) - + def __getitem__(self, name): if self.IsDirty(name): self.ParseNewFlags() @@ -208,3 +209,7 @@ DEFINE_string('node_availability_zone', 'nova', DEFINE_string('node_name', socket.gethostname(), 'name of this node') +DEFINE_string('sql_connection', + 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), + 'connection string for sql database') + diff --git a/nova/models.py b/nova/models.py index 7ad37981..36d6cf3a 100644 --- a/nova/models.py +++ b/nova/models.py @@ -19,7 +19,6 @@ """ SQLAlchemy models for nova data """ -import os from sqlalchemy.orm import relationship, backref, validates, exc from sqlalchemy import Table, Column, Integer, String @@ -34,10 +33,6 @@ FLAGS=flags.FLAGS Base = declarative_base() -flags.DEFINE_string('sql_connection', - 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), - 'connection string for sql database') - class NovaBase(object): __table_args__ = {'mysql_engine':'InnoDB'} __prefix__ = 'none' diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index d8a398aa..c982b18d 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -75,6 +75,15 @@ class NetworkTestCase(test.TrialTestCase): self.manager.delete_project(project) self.manager.delete_user(self.user) + def _create_address(self, project_num, instance_id=None): + net = db.project_get_network(None, self.projects[project_num].id) + fixed_ip = db.fixed_ip_allocate(None, net['id']) + address = fixed_ip['str_id'] + if instance_id is None: + instance_id = self.instance_id + db.fixed_ip_instance_associate(None, address, instance_id) + return address + def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" # TODO(vish): better way of adding floating ips @@ -87,89 +96,82 @@ class NetworkTestCase(test.TrialTestCase): floating_ip.ip_str = ip_str floating_ip.node_name = FLAGS.node_name floating_ip.save() - eaddress = self.service.allocate_floating_ip(self.projects[0].id) - faddress = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) - self.assertEqual(eaddress, str(pubnet[0])) - self.service.associate_floating_ip(eaddress, faddress) + float_addr = self.service.allocate_floating_ip(self.projects[0].id) + fix_addr = self._create_address(0) + self.assertEqual(float_addr, str(pubnet[0])) + self.service.associate_floating_ip(float_addr, fix_addr) # FIXME datamodel abstraction - self.assertEqual(floating_ip.fixed_ip.ip_str, faddress) - self.service.disassociate_floating_ip(eaddress) - self.assertEqual(floating_ip.fixed_ip, None) - self.service.deallocate_floating_ip(eaddress) - self.service.deallocate_fixed_ip(faddress) + address = db.instance_get_floating_address(None, self.instance_id) + self.assertEqual(address, float_addr) + self.service.disassociate_floating_ip(float_addr) + address = db.instance_get_floating_address(None, self.instance_id) + self.assertEqual(address, None) + self.service.deallocate_floating_ip(float_addr) + db.fixed_ip_deallocate(None, fix_addr) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" - address = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) - net = db.project_get_network(None, self.projects[0].id) + address = self._create_address(0) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) - issue_ip(address, net.bridge) - self.service.deallocate_fixed_ip(address) + lease_ip(address) + db.fixed_ip_deallocate(None, address) # Doesn't go away until it's dhcp released self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) - release_ip(address, net.bridge) + release_ip(address) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) def test_side_effects(self): """Ensures allocating and releasing has no side effects""" - address = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) - address2 = self.service.allocate_fixed_ip(self.projects[1].id, - self.instance2_id) - - net = db.project_get_network(None, self.projects[0].id) - net2 = db.project_get_network(None, self.projects[1].id) + address = self._create_address(0) + address2 = self._create_address(1, self.instance2_id) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) self.assertFalse(is_allocated_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued - issue_ip(address, net.bridge) - issue_ip(address2, net2.bridge) + lease_ip(address) + lease_ip(address2) - self.service.deallocate_fixed_ip(address) - release_ip(address, net.bridge) + db.fixed_ip_deallocate(None, address) + release_ip(address) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) - self.service.deallocate_fixed_ip(address2) - release_ip(address2, net2.bridge) + db.fixed_ip_deallocate(None, address2) + release_ip(address2) self.assertFalse(is_allocated_in_project(address2, self.projects[1].id)) def test_subnet_edge(self): """Makes sure that private ips don't overlap""" - first = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) + first = self._create_address(0) + lease_ip(first) for i in range(1, 5): project_id = self.projects[i].id - address = self.service.allocate_fixed_ip(project_id, self.instance_id) - address2 = self.service.allocate_fixed_ip(project_id, self.instance_id) - address3 = self.service.allocate_fixed_ip(project_id, self.instance_id) - net = db.project_get_network(None, project_id) - issue_ip(address, net.bridge) - issue_ip(address2, net.bridge) - issue_ip(address3, net.bridge) + address = self._create_address(i) + address2 = self._create_address(i) + address3 = self._create_address(i) + lease_ip(address) + lease_ip(address2) + lease_ip(address3) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) self.assertFalse(is_allocated_in_project(address2, self.projects[0].id)) self.assertFalse(is_allocated_in_project(address3, self.projects[0].id)) - self.service.deallocate_fixed_ip(address) - self.service.deallocate_fixed_ip(address2) - self.service.deallocate_fixed_ip(address3) - release_ip(address, net.bridge) - release_ip(address2, net.bridge) - release_ip(address3, net.bridge) - net = db.project_get_network(None, self.projects[0].id) - self.service.deallocate_fixed_ip(first) + db.fixed_ip_deallocate(None, address) + db.fixed_ip_deallocate(None, address2) + db.fixed_ip_deallocate(None, address3) + release_ip(address) + release_ip(address2) + release_ip(address3) + release_ip(first) + db.fixed_ip_deallocate(None, first) def test_vpn_ip_and_port_looks_valid(self): """Ensure the vpn ip and port are reasonable""" @@ -196,17 +198,14 @@ class NetworkTestCase(test.TrialTestCase): def test_ips_are_reused(self): """Makes sure that ip addresses that are deallocated get reused""" - address = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) - net = db.project_get_network(None, self.projects[0].id) - issue_ip(address, net.bridge) - self.service.deallocate_fixed_ip(address) - release_ip(address, net.bridge) + address = self._create_address(0) + lease_ip(address) + db.fixed_ip_deallocate(None, address) + release_ip(address) - address2 = self.service.allocate_fixed_ip(self.projects[0].id, - self.instance_id) + address2 = self._create_address(0) self.assertEqual(address, address2) - self.service.deallocate_fixed_ip(address2) + db.fixed_ip_deallocate(None, address2) def test_available_ips(self): """Make sure the number of available ips for the network is correct @@ -237,19 +236,19 @@ class NetworkTestCase(test.TrialTestCase): addresses = [] for i in range(num_available_ips): project_id = self.projects[0].id - addresses.append(self.service.allocate_fixed_ip(project_id, - self.instance_id)) - issue_ip(addresses[i],network.bridge) + address = self._create_address(0) + addresses.append(address) + lease_ip(address) self.assertEqual(available_ips(network), 0) self.assertRaises(db.NoMoreAddresses, - self.service.allocate_fixed_ip, - self.projects[0].id, - self.instance_id) + db.fixed_ip_allocate, + None, + network['id']) for i in range(len(addresses)): - self.service.deallocate_fixed_ip(addresses[i]) - release_ip(addresses[i],network.bridge) + db.fixed_ip_deallocate(None, addresses[i]) + release_ip(addresses[i]) self.assertEqual(available_ips(network), num_available_ips) @@ -287,20 +286,22 @@ def binpath(script): return os.path.abspath(os.path.join(__file__, "../../../bin", script)) -def issue_ip(private_ip, interface): +def lease_ip(private_ip): """Run add command on dhcpbridge""" + network_ref = db.fixed_ip_get_network(None, private_ip) cmd = "%s add fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) - env = {'DNSMASQ_INTERFACE': interface, + env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s ", out, err) -def release_ip(private_ip, interface): +def release_ip(private_ip): """Run del command on dhcpbridge""" + network_ref = db.fixed_ip_get_network(None, private_ip) cmd = "%s del fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) - env = {'DNSMASQ_INTERFACE': interface, + env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) From 3917a214704207aa5ca78326966b09d0e7acf5cd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 26 Aug 2010 11:28:05 -0700 Subject: [PATCH 049/113] fixed volume unit tests --- nova/tests/fake_flags.py | 2 ++ nova/tests/volume_unittest.py | 63 +++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 7fc83bab..543641a1 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -25,6 +25,8 @@ FLAGS.fake_storage = True FLAGS.fake_rabbit = True FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' +FLAGS.network_size = 16 +FLAGS.num_networks = 5 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' #FLAGS.sql_connection = 'mysql://root@localhost/test' diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 37ee6c72..e6b7b07c 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -55,12 +55,20 @@ class VolumeTestCase(test.TrialTestCase): for device in self.devices: device.delete() + def _create_volume(self, size='0'): + vol = {} + vol['size'] = '0' + vol['user_id'] = 'fake' + vol['project_id'] = 'fake' + vol['availability_zone'] = FLAGS.storage_availability_zone + vol['status'] = "creating" + vol['attach_status'] = "detached" + return db.volume_create(None, vol) + @defer.inlineCallbacks def test_run_create_volume(self): - vol_size = '0' - user_id = 'fake' - project_id = 'fake' - volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) + volume_id = self._create_volume() + yield self.volume.create_volume(volume_id) self.assertEqual(volume_id, models.Volume.find(volume_id).id) @@ -69,28 +77,27 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_big_volume(self): - vol_size = '1001' - user_id = 'fake' - project_id = 'fake' + # FIXME(vish): validation needs to move into the data layer in + # volume_create + defer.returnValue(True) try: - yield self.volume.create_volume(vol_size, user_id, project_id) + volume_id = self._create_volume('1001') + yield self.volume.create_volume(volume_id) self.fail("Should have thrown TypeError") except TypeError: pass @defer.inlineCallbacks def test_too_many_volumes(self): - vol_size = '1' - user_id = 'fake' - project_id = 'fake' vols = [] for i in xrange(self.total_slots): - vid = yield self.volume.create_volume(vol_size, user_id, project_id) - vols.append(vid) - self.assertFailure(self.volume.create_volume(vol_size, - user_id, - project_id), + volume_id = self._create_volume() + yield self.volume.create_volume(volume_id) + vols.append(volume_id) + volume_id = self._create_volume() + self.assertFailure(self.volume.create_volume(volume_id), db.NoMoreBlades) + db.volume_destroy(None, volume_id) for id in vols: yield self.volume.delete_volume(id) @@ -98,11 +105,9 @@ class VolumeTestCase(test.TrialTestCase): def test_run_attach_detach_volume(self): # Create one volume and one compute to test with instance_id = "storage-test" - vol_size = "5" - user_id = "fake" - project_id = 'fake' mountpoint = "/dev/sdf" - volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) + volume_id = self._create_volume() + yield self.volume.create_volume(volume_id) if FLAGS.fake_tests: db.volume_attached(None, volume_id, instance_id, mountpoint) else: @@ -110,10 +115,10 @@ class VolumeTestCase(test.TrialTestCase): volume_id, mountpoint) vol = db.volume_get(None, volume_id) - self.assertEqual(vol.status, "in-use") - self.assertEqual(vol.attach_status, "attached") - self.assertEqual(vol.instance_id, instance_id) - self.assertEqual(vol.mountpoint, mountpoint) + self.assertEqual(vol['status'], "in-use") + self.assertEqual(vol['attach_status'], "attached") + self.assertEqual(vol['instance_id'], instance_id) + self.assertEqual(vol['mountpoint'], mountpoint) self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) if FLAGS.fake_tests: @@ -121,11 +126,12 @@ class VolumeTestCase(test.TrialTestCase): else: rv = yield self.volume.detach_volume(instance_id, volume_id) - self.assertEqual(vol.status, "available") + self.assertEqual(vol['status'], "available") rv = self.volume.delete_volume(volume_id) self.assertRaises(exception.Error, - models.Volume.find, + db.volume_get, + None, volume_id) @defer.inlineCallbacks @@ -137,7 +143,7 @@ class VolumeTestCase(test.TrialTestCase): volume_ids = [] def _check(volume_id): volume_ids.append(volume_id) - vol = models.Volume.find(volume_id) + vol = db.volume_get(None, volume_id) shelf_blade = '%s.%s' % (vol.export_device.shelf_id, vol.export_device.blade_id) self.assert_(shelf_blade not in shelf_blades) @@ -145,7 +151,8 @@ class VolumeTestCase(test.TrialTestCase): logging.debug("got %s" % shelf_blade) deferreds = [] for i in range(self.total_slots): - d = self.volume.create_volume(vol_size, user_id, project_id) + volume_id = self._create_volume() + d = self.volume.create_volume(volume_id) d.addCallback(_check) d.addErrback(self.fail) deferreds.append(d) From 1a1a21294555cbb00c1f5bc251f983b37b002eef Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 26 Aug 2010 12:56:07 -0700 Subject: [PATCH 050/113] fixed service mox test cases --- nova/tests/service_unittest.py | 54 ++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 48298846..0b9d6002 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -80,13 +80,15 @@ class ServiceTestCase(test.BaseTestCase): binary = 'bar' daemon_ref = {'node_name': node_name, 'binary': binary, - 'report_count': 0 - } + 'report_count': 0, + 'id': 1} - service.db.daemon_get(None, node_name, binary).AndReturn(daemon_ref) - service.db.daemon_update(None, node_name, binary, + service.db.daemon_get_by_args(None, + node_name, + binary).AndReturn(daemon_ref) + service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) - + self.mox.ReplayAll() s = service.Service() rv = yield s.report_state(node_name, binary) @@ -95,17 +97,22 @@ class ServiceTestCase(test.BaseTestCase): def test_report_state_no_daemon(self): node_name = 'foo' binary = 'bar' + daemon_create = {'node_name': node_name, + 'binary': binary, + 'report_count': 0} daemon_ref = {'node_name': node_name, 'binary': binary, - 'report_count': 0 - } + 'report_count': 0, + 'id': 1} - service.db.daemon_get(None, node_name, binary).AndRaise( - exception.NotFound()) - service.db.daemon_create(None, daemon_ref).AndReturn(daemon_ref) - service.db.daemon_update(None, node_name, binary, + service.db.daemon_get_by_args(None, + node_name, + binary).AndRaise(exception.NotFound()) + service.db.daemon_create(None, daemon_create).AndReturn(daemon_ref['id']) + service.db.daemon_get(None, daemon_ref['id']).AndReturn(daemon_ref) + service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) - + self.mox.ReplayAll() s = service.Service() rv = yield s.report_state(node_name, binary) @@ -116,12 +123,13 @@ class ServiceTestCase(test.BaseTestCase): binary = 'bar' daemon_ref = {'node_name': node_name, 'binary': binary, - 'report_count': 0 - } + 'report_count': 0, + 'id': 1} + + service.db.daemon_get_by_args(None, + node_name, + binary).AndRaise(Exception()) - service.db.daemon_get(None, node_name, binary).AndRaise( - Exception()) - self.mox.ReplayAll() s = service.Service() rv = yield s.report_state(node_name, binary) @@ -134,13 +142,15 @@ class ServiceTestCase(test.BaseTestCase): binary = 'bar' daemon_ref = {'node_name': node_name, 'binary': binary, - 'report_count': 0 - } + 'report_count': 0, + 'id': 1} - service.db.daemon_get(None, node_name, binary).AndReturn(daemon_ref) - service.db.daemon_update(None, node_name, binary, + service.db.daemon_get_by_args(None, + node_name, + binary).AndReturn(daemon_ref) + service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) - + self.mox.ReplayAll() s = service.Service() s.model_disconnected = True From aad4702efe53aaf2fe5819f14dad510bcaed5217 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 27 Aug 2010 13:45:05 -0700 Subject: [PATCH 051/113] removed the last few references to models.py --- nova/tests/compute_unittest.py | 39 +++++++++++++-------------- nova/tests/fake_flags.py | 2 ++ nova/tests/network_unittest.py | 48 +++++++++------------------------- nova/tests/volume_unittest.py | 30 +++++++-------------- 4 files changed, 43 insertions(+), 76 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 44cc6ac1..e8597383 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -21,11 +21,11 @@ import time from twisted.internet import defer from xml.etree import ElementTree +from nova import db from nova import exception from nova import flags from nova import test from nova import utils -from nova import models from nova.auth import manager from nova.compute import service @@ -69,47 +69,44 @@ class ComputeConnectionTestCase(test.TrialTestCase): self.manager.delete_user('fake') self.manager.delete_project('fake') - def create_instance(self): - inst = models.Instance(user_id='fake', project_id='fake', image_id='ami-test') - inst.save(); - # TODO(ja): add ami, ari, aki, user_data - # inst['reservation_id'] = 'r-fakeres' - # inst['launch_time'] = '10' - #inst['user_id'] = 'fake' - #inst['project_id'] = 'fake' - #inst['instance_type'] = 'm1.tiny' - #inst['node_name'] = FLAGS.node_name - #inst['mac_address'] = utils.generate_mac() - #inst['ami_launch_index'] = 0 - #inst.save() - return inst.id + def _create_instance(self): + inst = {} + inst['image_id'] = 'ami-test' + inst['reservation_id'] = 'r-fakeres' + inst['launch_time'] = '10' + inst['user_id'] = 'fake' + inst['project_id'] = 'fake' + inst['instance_type'] = 'm1.tiny' + inst['mac_address'] = utils.generate_mac() + inst['ami_launch_index'] = 0 + return db.instance_create(None, inst) @defer.inlineCallbacks def test_run_describe_terminate(self): - instance_id = self.create_instance() + instance_id = self._create_instance() yield self.compute.run_instance(instance_id) - instances = models.Instance.all() + instances = db.instance_get_all(None) logging.info("Running instances: %s", instances) self.assertEqual(len(instances), 1) yield self.compute.terminate_instance(instance_id) - instances = models.Instance.all() + instances = db.instance_get_all(None) logging.info("After terminating instances: %s", instances) self.assertEqual(len(instances), 0) @defer.inlineCallbacks def test_reboot(self): - instance_id = self.create_instance() + instance_id = self._create_instance() yield self.compute.run_instance(instance_id) yield self.compute.reboot_instance(instance_id) yield self.compute.terminate_instance(instance_id) @defer.inlineCallbacks def test_console_output(self): - instance_id = self.create_instance() + instance_id = self._create_instance() rv = yield self.compute.run_instance(instance_id) console = yield self.compute.get_console_output(instance_id) @@ -118,7 +115,7 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_instance_existing(self): - instance_id = self.create_instance() + instance_id = self._create_instance() yield self.compute.run_instance(instance_id) self.assertFailure(self.compute.run_instance(instance_id), exception.Error) yield self.compute.terminate_instance(instance_id) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 543641a1..42a13e4e 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -27,6 +27,8 @@ FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' FLAGS.network_size = 16 FLAGS.num_networks = 5 +FLAGS.num_shelves = 2 +FLAGS.blades_per_shelf = 4 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' #FLAGS.sql_connection = 'mysql://root@localhost/test' diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index c982b18d..d487c2e4 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -25,7 +25,6 @@ import logging from nova import db from nova import exception from nova import flags -from nova import models from nova import test from nova import utils from nova.auth import manager @@ -90,17 +89,13 @@ class NetworkTestCase(test.TrialTestCase): pubnet = IPy.IP(flags.FLAGS.public_range) ip_str = str(pubnet[0]) try: - floating_ip = models.FloatingIp.find_by_str(ip_str) + db.floating_ip_get_by_address(None, ip_str) except exception.NotFound: - floating_ip = models.FloatingIp() - floating_ip.ip_str = ip_str - floating_ip.node_name = FLAGS.node_name - floating_ip.save() + db.floating_ip_create(None, ip_str, FLAGS.node_name) float_addr = self.service.allocate_floating_ip(self.projects[0].id) fix_addr = self._create_address(0) self.assertEqual(float_addr, str(pubnet[0])) self.service.associate_floating_ip(float_addr, fix_addr) - # FIXME datamodel abstraction address = db.instance_get_floating_address(None, self.instance_id) self.assertEqual(address, float_addr) self.service.disassociate_floating_ip(float_addr) @@ -183,8 +178,7 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_networks(self): """Ensure error is raised if we run out of networks""" projects = [] - # TODO(vish): use data layer for count - networks_left = FLAGS.num_networks - models.Network.count() + networks_left = FLAGS.num_networks - db.network_count(None) for i in range(networks_left): project = self.manager.create_project('many%s' % i, self.user) projects.append(project) @@ -220,9 +214,9 @@ class NetworkTestCase(test.TrialTestCase): """ network = db.project_get_network(None, self.projects[0].id) net_size = flags.FLAGS.network_size - total_ips = (available_ips(network) + - reserved_ips(network) + - allocated_ips(network)) + total_ips = (db.network_count_available_ips(None, network['id']) + + db.network_count_reserved_ips(None, network['id']) + + db.network_count_allocated_ips(None, network['id'])) self.assertEqual(total_ips, net_size) def test_too_many_addresses(self): @@ -230,9 +224,9 @@ class NetworkTestCase(test.TrialTestCase): """ network = db.project_get_network(None, self.projects[0].id) - # Number of availaible ips is len of the available list - num_available_ips = available_ips(network) + num_available_ips = db.network_count_available_ips(None, + network['id']) addresses = [] for i in range(num_available_ips): project_id = self.projects[0].id @@ -240,7 +234,8 @@ class NetworkTestCase(test.TrialTestCase): addresses.append(address) lease_ip(address) - self.assertEqual(available_ips(network), 0) + self.assertEqual(db.network_count_available_ips(None, + network['id']), 0) self.assertRaises(db.NoMoreAddresses, db.fixed_ip_allocate, None, @@ -249,28 +244,11 @@ class NetworkTestCase(test.TrialTestCase): for i in range(len(addresses)): db.fixed_ip_deallocate(None, addresses[i]) release_ip(addresses[i]) - self.assertEqual(available_ips(network), num_available_ips) + self.assertEqual(db.network_count_available_ips(None, + network['id']), + num_available_ips) -# FIXME move these to abstraction layer -def available_ips(network): - session = models.NovaBase.get_session() - query = session.query(models.FixedIp).filter_by(network_id=network.id) - query = query.filter_by(allocated=False).filter_by(reserved=False) - return query.count() - -def allocated_ips(network): - session = models.NovaBase.get_session() - query = session.query(models.FixedIp).filter_by(network_id=network.id) - query = query.filter_by(allocated=True) - return query.count() - -def reserved_ips(network): - session = models.NovaBase.get_session() - query = session.query(models.FixedIp).filter_by(network_id=network.id) - query = query.filter_by(reserved=True) - return query.count() - def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" fixed_ip = db.fixed_ip_get_by_address(None, address) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index e6b7b07c..a03e0e6e 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -23,7 +23,6 @@ from twisted.internet import defer from nova import exception from nova import db from nova import flags -from nova import models from nova import test from nova.compute import service as compute_service from nova.volume import service as volume_service @@ -40,20 +39,7 @@ class VolumeTestCase(test.TrialTestCase): self.flags(connection_type='fake', fake_storage=True) self.volume = volume_service.VolumeService() - self.total_slots = 10 - # FIXME this should test actual creation method - self.devices = [] - for i in xrange(self.total_slots): - export_device = models.ExportDevice() - export_device.shelf_id = 0 - export_device.blade_id = i - export_device.save() - self.devices.append(export_device) - def tearDown(self): - super(VolumeTestCase, self).tearDown() - for device in self.devices: - device.delete() def _create_volume(self, size='0'): vol = {} @@ -69,11 +55,13 @@ class VolumeTestCase(test.TrialTestCase): def test_run_create_volume(self): volume_id = self._create_volume() yield self.volume.create_volume(volume_id) - self.assertEqual(volume_id, - models.Volume.find(volume_id).id) + self.assertEqual(volume_id, db.volume_get(None, volume_id).id) yield self.volume.delete_volume(volume_id) - self.assertRaises(exception.NotFound, models.Volume.find, volume_id) + self.assertRaises(exception.NotFound, + db.volume_get, + None, + volume_id) @defer.inlineCallbacks def test_too_big_volume(self): @@ -90,7 +78,8 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): vols = [] - for i in xrange(self.total_slots): + total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + for i in xrange(total_slots): volume_id = self._create_volume() yield self.volume.create_volume(volume_id) vols.append(volume_id) @@ -150,7 +139,8 @@ class VolumeTestCase(test.TrialTestCase): shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) deferreds = [] - for i in range(self.total_slots): + total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + for i in range(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(volume_id) d.addCallback(_check) @@ -158,7 +148,7 @@ class VolumeTestCase(test.TrialTestCase): deferreds.append(d) yield defer.DeferredList(deferreds) for volume_id in volume_ids: - vol = models.Volume.find(volume_id) + vol = db.volume_get(None, volume_id) vol.delete() def test_multi_node(self): From 397a762fdddb401b449cd9aa92a2a08f3801248a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 27 Aug 2010 13:46:27 -0700 Subject: [PATCH 052/113] moved models.py --- nova/models.py | 368 ------------------------------------------------- 1 file changed, 368 deletions(-) delete mode 100644 nova/models.py diff --git a/nova/models.py b/nova/models.py deleted file mode 100644 index 36d6cf3a..00000000 --- a/nova/models.py +++ /dev/null @@ -1,368 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -SQLAlchemy models for nova data -""" - -from sqlalchemy.orm import relationship, backref, validates, exc -from sqlalchemy import Table, Column, Integer, String -from sqlalchemy import MetaData, ForeignKey, DateTime, Boolean, Text -from sqlalchemy.ext.declarative import declarative_base - -from nova import auth -from nova import exception -from nova import flags - -FLAGS=flags.FLAGS - -Base = declarative_base() - -class NovaBase(object): - __table_args__ = {'mysql_engine':'InnoDB'} - __prefix__ = 'none' - created_at = Column(DateTime) - updated_at = Column(DateTime) - - _session = None - _engine = None - @classmethod - def create_engine(cls): - if NovaBase._engine is not None: - return NovaBase._engine - from sqlalchemy import create_engine - NovaBase._engine = create_engine(FLAGS.sql_connection, echo=False) - Base.metadata.create_all(NovaBase._engine) - return NovaBase._engine - - @classmethod - def get_session(cls): - from sqlalchemy.orm import sessionmaker - if NovaBase._session == None: - NovaBase.create_engine() - NovaBase._session = sessionmaker(bind=NovaBase._engine)() - return NovaBase._session - - @classmethod - def all(cls): - session = NovaBase.get_session() - result = session.query(cls).all() - session.commit() - return result - - @classmethod - def count(cls): - session = NovaBase.get_session() - result = session.query(cls).count() - session.commit() - return result - - @classmethod - def find(cls, obj_id): - session = NovaBase.get_session() - try: - result = session.query(cls).filter_by(id=obj_id).one() - session.commit() - return result - except exc.NoResultFound: - raise exception.NotFound("No model for id %s" % obj_id) - - @classmethod - def find_by_str(cls, str_id): - id = int(str_id.rpartition('-')[2]) - return cls.find(id) - - @property - def str_id(self): - return "%s-%s" % (self.__prefix__, self.id) - - def save(self): - session = NovaBase.get_session() - session.add(self) - session.commit() - - def delete(self): - session = NovaBase.get_session() - session.delete(self) - session.commit() - - def refresh(self): - session = NovaBase.get_session() - session.refresh(self) - - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - -class Image(Base, NovaBase): - __tablename__ = 'images' - __prefix__ = 'ami' - id = Column(Integer, primary_key=True) - user_id = Column(String(255))#, ForeignKey('users.id'), nullable=False) - project_id = Column(String(255))#, ForeignKey('projects.id'), nullable=False) - - image_type = Column(String(255)) - public = Column(Boolean, default=False) - state = Column(String(255)) - location = Column(String(255)) - arch = Column(String(255)) - default_kernel_id = Column(String(255)) - default_ramdisk_id = Column(String(255)) - - @validates('image_type') - def validate_image_type(self, key, image_type): - assert(image_type in ['machine', 'kernel', 'ramdisk', 'raw']) - - @validates('state') - def validate_state(self, key, state): - assert(state in ['available', 'pending', 'disabled']) - - @validates('default_kernel_id') - def validate_kernel_id(self, key, val): - if val != 'machine': - assert(val is None) - - @validates('default_ramdisk_id') - def validate_ramdisk_id(self, key, val): - if val != 'machine': - assert(val is None) - - -class PhysicalNode(Base, NovaBase): - __tablename__ = 'physical_nodes' - id = Column(String(255), primary_key=True) - -class Daemon(Base, NovaBase): - __tablename__ = 'daemons' - id = Column(Integer, primary_key=True) - node_name = Column(String(255)) #, ForeignKey('physical_node.id')) - binary = Column(String(255)) - report_count = Column(Integer, nullable=False, default=0) - - @classmethod - def find_by_args(cls, node_name, binary): - session = NovaBase.get_session() - try: - query = session.query(cls).filter_by(node_name=node_name) - result = query.filter_by(binary=binary).one() - session.commit() - return result - except exc.NoResultFound: - raise exception.NotFound("No model for %s, %s" % (node_name, - binary)) - - -class Instance(Base, NovaBase): - __tablename__ = 'instances' - __prefix__ = 'i' - id = Column(Integer, primary_key=True) - - user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) - project_id = Column(String(255)) #, ForeignKey('projects.id')) - - @property - def user(self): - return auth.manager.AuthManager().get_user(self.user_id) - - @property - def project(self): - return auth.manager.AuthManager().get_project(self.project_id) - - # TODO(vish): make this opaque somehow - @property - def name(self): - return self.str_id - - - image_id = Column(Integer, ForeignKey('images.id'), nullable=True) - kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) - ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) - - launch_index = Column(Integer) - key_name = Column(String(255)) - key_data = Column(Text) - security_group = Column(String(255)) - - state = Column(Integer) - state_description = Column(String(255)) - - hostname = Column(String(255)) - node_name = Column(String(255)) #, ForeignKey('physical_node.id')) - - instance_type = Column(Integer) - - user_data = Column(Text) - - reservation_id = Column(String(255)) - mac_address = Column(String(255)) - - def set_state(self, state_code, state_description=None): - from nova.compute import power_state - self.state = state_code - if not state_description: - state_description = power_state.name(state_code) - self.state_description = state_description - self.save() - -# ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) -# kernel = relationship(Kernel, backref=backref('instances', order_by=id)) -# project = relationship(Project, backref=backref('instances', order_by=id)) - -#TODO - see Ewan's email about state improvements - # vmstate_state = running, halted, suspended, paused - # power_state = what we have - # task_state = transitory and may trigger power state transition - - #@validates('state') - #def validate_state(self, key, state): - # assert(state in ['nostate', 'running', 'blocked', 'paused', 'shutdown', 'shutoff', 'crashed']) - -class Volume(Base, NovaBase): - __tablename__ = 'volumes' - __prefix__ = 'vol' - id = Column(Integer, primary_key=True) - - user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False) - project_id = Column(String(255)) #, ForeignKey('projects.id')) - - node_name = Column(String(255)) #, ForeignKey('physical_node.id')) - size = Column(Integer) - availability_zone = Column(String(255)) # TODO(vish) foreign key? - instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) - mountpoint = Column(String(255)) - attach_time = Column(String(255)) # TODO(vish) datetime - status = Column(String(255)) # TODO(vish) enum? - attach_status = Column(String(255)) # TODO(vish) enum - -class ExportDevice(Base, NovaBase): - __tablename__ = 'export_devices' - id = Column(Integer, primary_key=True) - shelf_id = Column(Integer) - blade_id = Column(Integer) - volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) - volume = relationship(Volume, backref=backref('export_device', - uselist=False)) - - -# TODO(vish): can these both come from the same baseclass? -class FixedIp(Base, NovaBase): - __tablename__ = 'fixed_ips' - id = Column(Integer, primary_key=True) - ip_str = Column(String(255), unique=True) - network_id = Column(Integer, ForeignKey('networks.id'), nullable=False) - instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) - instance = relationship(Instance, backref=backref('fixed_ip', - uselist=False)) - allocated = Column(Boolean, default=False) - leased = Column(Boolean, default=False) - reserved = Column(Boolean, default=False) - - @property - def str_id(self): - return self.ip_str - - @classmethod - def find_by_str(cls, str_id): - session = NovaBase.get_session() - try: - result = session.query(cls).filter_by(ip_str=str_id).one() - session.commit() - return result - except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % str_id) - - -class FloatingIp(Base, NovaBase): - __tablename__ = 'floating_ips' - id = Column(Integer, primary_key=True) - ip_str = Column(String(255), unique=True) - fixed_ip_id = Column(Integer, ForeignKey('fixed_ips.id'), nullable=True) - fixed_ip = relationship(FixedIp, backref=backref('floating_ips')) - - project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) - node_name = Column(String(255)) #, ForeignKey('physical_node.id')) - - @property - def str_id(self): - return self.ip_str - - @classmethod - def find_by_str(cls, str_id): - session = NovaBase.get_session() - try: - result = session.query(cls).filter_by(ip_str=str_id).one() - session.commit() - return result - except exc.NoResultFound: - raise exception.NotFound("No model for ip str %s" % str_id) - - -class Network(Base, NovaBase): - __tablename__ = 'networks' - id = Column(Integer, primary_key=True) - kind = Column(String(255)) - - injected = Column(Boolean, default=False) - cidr = Column(String(255)) - netmask = Column(String(255)) - bridge = Column(String(255)) - gateway = Column(String(255)) - broadcast = Column(String(255)) - dns = Column(String(255)) - - vlan = Column(Integer) - vpn_public_ip_str = Column(String(255)) - vpn_public_port = Column(Integer) - vpn_private_ip_str = Column(String(255)) - dhcp_start = Column(String(255)) - - project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False) - node_name = Column(String(255)) #, ForeignKey('physical_node.id')) - - fixed_ips = relationship(FixedIp, - single_parent=True, - backref=backref('network'), - cascade='all, delete, delete-orphan') - - -class NetworkIndex(Base, NovaBase): - __tablename__ = 'network_indexes' - id = Column(Integer, primary_key=True) - index = Column(Integer) - network_id = Column(Integer, ForeignKey('networks.id'), nullable=True) - network = relationship(Network, backref=backref('network_index', - uselist=False)) - - - - -def create_session(engine=None): - return NovaBase.get_session() - -if __name__ == '__main__': - engine = NovaBase.create_engine() - session = NovaBase.create_session(engine) - - instance = Instance(image_id='as', ramdisk_id='AS', user_id='anthony') - user = User(id='anthony') - session.add(instance) - session.commit() - From e90c1ac17a37942bf5479fc6e5a059f8516a9f15 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 27 Aug 2010 23:10:57 -0700 Subject: [PATCH 053/113] split volume into service/manager/driver --- nova/endpoint/cloud.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ffe3d3cc..6d59c822 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -271,7 +271,6 @@ class CloudController(object): return v @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): vol = {} vol['size'] = size @@ -280,13 +279,12 @@ class CloudController(object): vol['availability_zone'] = FLAGS.storage_availability_zone vol['status'] = "creating" vol['attach_status'] = "detached" - volume_id = db.volume_create(context, vol) + volume_ref = db.volume_create(context, vol) - yield rpc.cast(FLAGS.volume_topic, {"method": "create_volume", - "args": {"volume_id": volume_id}}) + rpc.cast(FLAGS.volume_topic, {"method": "create_volume", + "args": {"volume_id": volume_ref['id']}}) - volume = db.volume_get(context, volume_id) - defer.returnValue({'volumeSet': [self._format_volume(context, volume)]}) + return {'volumeSet': [self._format_volume(context, volume_ref)]} @rbac.allow('projectmanager', 'sysadmin') From cb3721e2fee9a8694623a158e14c3955b89555ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 27 Aug 2010 23:16:31 -0700 Subject: [PATCH 054/113] move None context up into cloud --- nova/endpoint/cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 6d59c822..cb676aea 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -282,7 +282,8 @@ class CloudController(object): volume_ref = db.volume_create(context, vol) rpc.cast(FLAGS.volume_topic, {"method": "create_volume", - "args": {"volume_id": volume_ref['id']}}) + "args": {"context": None, + "volume_id": volume_ref['id']}}) return {'volumeSet': [self._format_volume(context, volume_ref)]} @@ -633,7 +634,8 @@ class CloudController(object): host = db.volume_get_host(context, volume_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "delete_volume", - "args": {"volume_id": volume_id}}) + "args": {"context": None, + "volume_id": volume_id}}) return defer.succeed(True) @rbac.allow('all') From 9e2de6b3ce78fed91e731eab570833efda50ab9b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 28 Aug 2010 02:02:07 -0700 Subject: [PATCH 055/113] moved network code into business layer --- nova/endpoint/cloud.py | 19 ++++++++----------- nova/flags.py | 8 ++++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index cb676aea..ceff0f82 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -60,6 +60,7 @@ class CloudController(object): sent to the other nodes. """ def __init__(self): + self.network_manager = utils.load_object(FLAGS.network_manager) self.setup() def __str__(self): @@ -522,7 +523,6 @@ class CloudController(object): # TODO: Get the real security group of launch in here security_group = "default" - network_ref = db.project_get_network(context, context.project.id) reservation_id = utils.generate_uid('r') base_options = {} base_options['image_id'] = image_id @@ -540,30 +540,27 @@ class CloudController(object): for num in range(int(kwargs['max_count'])): inst_id = db.instance_create(context, base_options) - if vpn: - fixed_ip = db.network_get_vpn_ip(context, network_ref['id']) - else: - fixed_ip = db.fixed_ip_allocate(context, network_ref['id']) - print fixed_ip['ip_str'], inst_id - db.fixed_ip_instance_associate(context, fixed_ip['ip_str'], inst_id) - print fixed_ip.instance inst = {} inst['mac_address'] = utils.generate_mac() inst['launch_index'] = num inst['hostname'] = inst_id db.instance_update(context, inst_id, inst) - + address = self.network_manager.allocate_fixed_ip(context, + inst_id, + vpn) # TODO(vish): This probably should be done in the scheduler # network is setup when host is assigned network_topic = yield self._get_network_topic(context) rpc.call(network_topic, {"method": "setup_fixed_ip", - "args": {"address": fixed_ip['ip_str']}}) + "args": {"context": None, + "address": address}}) rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id": inst_id}}) + "args": {"context": None, + "instance_id": inst_id}}) logging.debug("Casting to node for %s/%s's instance %s" % (context.project.name, context.user.name, inst_id)) defer.returnValue(self._format_run_instances(context, diff --git a/nova/flags.py b/nova/flags.py index d4b2b7c3..dfdfe978 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -213,3 +213,11 @@ DEFINE_string('sql_connection', 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), 'connection string for sql database') +DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', + 'Manager for compute') +DEFINE_string('network_manager', 'nova.network.manager.VlanManager', + 'Manager for network') +DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager', + 'Manager for volume') + + From 78ff2f216d33df23910ac299254910d22240964f Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Sat, 28 Aug 2010 23:06:40 -0700 Subject: [PATCH 056/113] Making tests pass --- nova/tests/network_unittest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index d487c2e4..e0de04be 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -105,6 +105,7 @@ class NetworkTestCase(test.TrialTestCase): db.fixed_ip_deallocate(None, fix_addr) def test_allocate_deallocate_fixed_ip(self): + import pdb; pdb.set_trace() """Makes sure that we can allocate and deallocate a fixed ip""" address = self._create_address(0) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) From ad7b34ecb32f011084e8995c4c69fb87fe799591 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 29 Aug 2010 18:53:47 -0700 Subject: [PATCH 057/113] tests pass --- bin/nova-dhcpbridge | 21 ++++++++++-------- nova/auth/manager.py | 4 +++- nova/endpoint/cloud.py | 2 +- nova/tests/cloud_unittest.py | 4 ++-- nova/tests/compute_unittest.py | 3 +-- nova/tests/fake_flags.py | 10 +++++++-- nova/tests/network_unittest.py | 14 ++++++------ nova/tests/service_unittest.py | 19 ++++++++++++----- nova/tests/volume_unittest.py | 39 +++++++++++++++++----------------- 9 files changed, 69 insertions(+), 47 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 6747a3a0..52ec2d49 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -34,23 +34,23 @@ from nova import db from nova import flags from nova import rpc from nova import utils -from nova.network import linux_net -from nova.network import service from nova import datastore # for redis_db flag from nova.auth import manager # for auth flags +from nova.network import manager # for network flags FLAGS = flags.FLAGS - def add_lease(_mac, ip_address, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: logging.debug("leasing ip") - service.VlanNetworkService().lease_fixed_ip(ip_address) + network_manager = utils.import_object(FLAGS.network_manager) + network_manager.lease_fixed_ip(None, ip_address) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "lease_fixed_ip", - "args": {"address": ip_address}}) + "args": {"context": None, + "address": ip_address}}) def old_lease(_mac, _ip_address, _hostname, _interface): @@ -62,20 +62,24 @@ def del_lease(_mac, ip_address, _hostname, _interface): """Called when a lease expires.""" if FLAGS.fake_rabbit: logging.debug("releasing ip") - service.VlanNetworkService().release_fixed_ip(ip_address) + network_manager = utils.import_object(FLAGS.network_manager) + network_manager.release_fixed_ip(None, ip_address) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "release_fixed_ip", - "args": {"address": ip_address}}) + "args": {"context": None, + "address": ip_address}}) def init_leases(interface): """Get the list of hosts for an interface.""" network_ref = db.network_get_by_bridge(None, interface) - return linux_net.get_dhcp_hosts(None, network_ref['id']) + network_manager = utils.import_object(FLAGS.network_manager) + return network_manager.driver.get_dhcp_hosts(None, network_ref['id']) def main(): + global network_manager """Parse environment and arguments and call the approproate action.""" flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) @@ -93,7 +97,6 @@ def main(): '..', '_trial_temp', 'nova.sqlite')) - print path FLAGS.sql_connection = 'sqlite:///%s' % path #FLAGS.sql_connection = 'mysql://root@localhost/test' action = argv[1] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index a072a143..62ec3f4e 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -252,6 +252,7 @@ class AuthManager(object): __init__ is run every time AuthManager() is called, so we only reset the driver if it is not set or a new driver is specified. """ + self.network_manager = utils.import_object(FLAGS.network_manager) if driver or not getattr(self, 'driver', None): self.driver = utils.import_class(driver or FLAGS.auth_driver) @@ -525,7 +526,8 @@ class AuthManager(object): if project_dict: project = Project(**project_dict) try: - db.network_allocate(context, project.id) + self.network_manager.allocate_network(context, + project.id) except: drv.delete_project(project.id) raise diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ceff0f82..8ba10a5b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -60,7 +60,7 @@ class CloudController(object): sent to the other nodes. """ def __init__(self): - self.network_manager = utils.load_object(FLAGS.network_manager) + self.network_manager = utils.import_object(FLAGS.network_manager) self.setup() def __str__(self): diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 3501771c..df2246aa 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -27,8 +27,8 @@ from xml.etree import ElementTree from nova import flags from nova import rpc from nova import test +from nova import utils from nova.auth import manager -from nova.compute import service from nova.endpoint import api from nova.endpoint import cloud @@ -53,7 +53,7 @@ class CloudTestCase(test.BaseTestCase): self.injected.append(self.cloud_consumer.attach_to_tornado(self.ioloop)) # set up a service - self.compute = service.ComputeService() + self.compute = utils.import_class(FLAGS.compute_manager) self.compute_consumer = rpc.AdapterConsumer(connection=self.conn, topic=FLAGS.compute_topic, proxy=self.compute) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index e8597383..28e51f38 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -27,7 +27,6 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager -from nova.compute import service FLAGS = flags.FLAGS @@ -60,7 +59,7 @@ class ComputeConnectionTestCase(test.TrialTestCase): super(ComputeConnectionTestCase, self).setUp() self.flags(connection_type='fake', fake_storage=True) - self.compute = service.ComputeService() + self.compute = utils.import_object(FLAGS.compute_manager) self.manager = manager.AuthManager() user = self.manager.create_user('fake', 'fake', 'fake') project = self.manager.create_project('fake', 'fake', 'fake') diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 42a13e4e..3114912b 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -20,13 +20,19 @@ from nova import flags FLAGS = flags.FLAGS -FLAGS.connection_type = 'fake' +flags.DECLARE('fake_storage', 'nova.volume.manager') FLAGS.fake_storage = True +FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True -FLAGS.fake_network = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' +flags.DECLARE('network_size', 'nova.network.manager') +flags.DECLARE('num_networks', 'nova.network.manager') +flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 16 FLAGS.num_networks = 5 +FLAGS.fake_network = True +flags.DECLARE('num_shelves', 'nova.volume.manager') +flags.DECLARE('blades_per_shelf', 'nova.volume.manager') FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.verbose = True diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index d487c2e4..e3fe01fa 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -49,14 +49,15 @@ class NetworkTestCase(test.TrialTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] - self.service = service.VlanNetworkService() + self.network = utils.import_object(FLAGS.network_manager) + self.context = None for i in range(5): name = 'project%s' % i self.projects.append(self.manager.create_project(name, 'netuser', name)) # create the necessary network data for the project - self.service.set_network_host(self.projects[i].id) + self.network.set_network_host(self.context, self.projects[i].id) instance_id = db.instance_create(None, {'mac_address': utils.generate_mac()}) self.instance_id = instance_id @@ -92,16 +93,17 @@ class NetworkTestCase(test.TrialTestCase): db.floating_ip_get_by_address(None, ip_str) except exception.NotFound: db.floating_ip_create(None, ip_str, FLAGS.node_name) - float_addr = self.service.allocate_floating_ip(self.projects[0].id) + float_addr = self.network.allocate_floating_ip(self.context, + self.projects[0].id) fix_addr = self._create_address(0) self.assertEqual(float_addr, str(pubnet[0])) - self.service.associate_floating_ip(float_addr, fix_addr) + self.network.associate_floating_ip(self.context, float_addr, fix_addr) address = db.instance_get_floating_address(None, self.instance_id) self.assertEqual(address, float_addr) - self.service.disassociate_floating_ip(float_addr) + self.network.disassociate_floating_ip(self.context, float_addr) address = db.instance_get_floating_address(None, self.instance_id) self.assertEqual(address, None) - self.service.deallocate_floating_ip(float_addr) + self.network.deallocate_floating_ip(self.context, float_addr) db.fixed_ip_deallocate(None, fix_addr) def test_allocate_deallocate_fixed_ip(self): diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 0b9d6002..e13fe62d 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -30,10 +30,16 @@ from nova import flags from nova import rpc from nova import test from nova import service - +from nova import manager FLAGS = flags.FLAGS +flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager", + "Manager for testing") + +class FakeManager(manager.Manager): + """Fake manager for tests""" + pass class ServiceTestCase(test.BaseTestCase): """Test cases for rpc""" @@ -46,12 +52,12 @@ class ServiceTestCase(test.BaseTestCase): self.mox.StubOutWithMock( service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='run_tests.py', + topic='fake', proxy=mox.IsA(service.Service) ).AndReturn(rpc.AdapterConsumer) rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='run_tests.py.%s' % FLAGS.node_name, + topic='fake.%s' % FLAGS.node_name, proxy=mox.IsA(service.Service) ).AndReturn(rpc.AdapterConsumer) @@ -67,7 +73,7 @@ class ServiceTestCase(test.BaseTestCase): rpc.AdapterConsumer.attach_to_twisted() self.mox.ReplayAll() - app = service.Service.create() + app = service.Service.create(bin_name='nova-fake') self.assert_(app) # We're testing sort of weird behavior in how report_state decides @@ -82,7 +88,7 @@ class ServiceTestCase(test.BaseTestCase): 'binary': binary, 'report_count': 0, 'id': 1} - + service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, node_name, binary).AndReturn(daemon_ref) @@ -105,6 +111,7 @@ class ServiceTestCase(test.BaseTestCase): 'report_count': 0, 'id': 1} + service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, node_name, binary).AndRaise(exception.NotFound()) @@ -126,6 +133,7 @@ class ServiceTestCase(test.BaseTestCase): 'report_count': 0, 'id': 1} + service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, node_name, binary).AndRaise(Exception()) @@ -145,6 +153,7 @@ class ServiceTestCase(test.BaseTestCase): 'report_count': 0, 'id': 1} + service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, node_name, binary).AndReturn(daemon_ref) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index a03e0e6e..4504276e 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -24,8 +24,7 @@ from nova import exception from nova import db from nova import flags from nova import test -from nova.compute import service as compute_service -from nova.volume import service as volume_service +from nova import utils FLAGS = flags.FLAGS @@ -35,10 +34,11 @@ class VolumeTestCase(test.TrialTestCase): def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() - self.compute = compute_service.ComputeService() + self.compute = utils.import_object(FLAGS.compute_manager) self.flags(connection_type='fake', fake_storage=True) - self.volume = volume_service.VolumeService() + self.volume = utils.import_object(FLAGS.volume_manager) + self.context = None def _create_volume(self, size='0'): @@ -49,15 +49,15 @@ class VolumeTestCase(test.TrialTestCase): vol['availability_zone'] = FLAGS.storage_availability_zone vol['status'] = "creating" vol['attach_status'] = "detached" - return db.volume_create(None, vol) + return db.volume_create(None, vol)['id'] @defer.inlineCallbacks def test_run_create_volume(self): volume_id = self._create_volume() - yield self.volume.create_volume(volume_id) + yield self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(None, volume_id).id) - yield self.volume.delete_volume(volume_id) + yield self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.NotFound, db.volume_get, None, @@ -70,7 +70,7 @@ class VolumeTestCase(test.TrialTestCase): defer.returnValue(True) try: volume_id = self._create_volume('1001') - yield self.volume.create_volume(volume_id) + yield self.volume.create_volume(self.context, volume_id) self.fail("Should have thrown TypeError") except TypeError: pass @@ -81,14 +81,15 @@ class VolumeTestCase(test.TrialTestCase): total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf for i in xrange(total_slots): volume_id = self._create_volume() - yield self.volume.create_volume(volume_id) + yield self.volume.create_volume(self.context, volume_id) vols.append(volume_id) volume_id = self._create_volume() - self.assertFailure(self.volume.create_volume(volume_id), + self.assertFailure(self.volume.create_volume(self.context, + volume_id), db.NoMoreBlades) db.volume_destroy(None, volume_id) - for id in vols: - yield self.volume.delete_volume(id) + for volume_id in vols: + yield self.volume.delete_volume(self.context, volume_id) @defer.inlineCallbacks def test_run_attach_detach_volume(self): @@ -96,7 +97,7 @@ class VolumeTestCase(test.TrialTestCase): instance_id = "storage-test" mountpoint = "/dev/sdf" volume_id = self._create_volume() - yield self.volume.create_volume(volume_id) + yield self.volume.create_volume(self.context, volume_id) if FLAGS.fake_tests: db.volume_attached(None, volume_id, instance_id, mountpoint) else: @@ -109,15 +110,16 @@ class VolumeTestCase(test.TrialTestCase): self.assertEqual(vol['instance_id'], instance_id) self.assertEqual(vol['mountpoint'], mountpoint) - self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) + self.assertFailure(self.volume.delete_volume(self.context, volume_id), + exception.Error) if FLAGS.fake_tests: db.volume_detached(None, volume_id) else: - rv = yield self.volume.detach_volume(instance_id, + rv = yield self.compute.detach_volume(instance_id, volume_id) self.assertEqual(vol['status'], "available") - rv = self.volume.delete_volume(volume_id) + rv = self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.Error, db.volume_get, None, @@ -142,14 +144,13 @@ class VolumeTestCase(test.TrialTestCase): total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf for i in range(total_slots): volume_id = self._create_volume() - d = self.volume.create_volume(volume_id) + d = self.volume.create_volume(self.context, volume_id) d.addCallback(_check) d.addErrback(self.fail) deferreds.append(d) yield defer.DeferredList(deferreds) for volume_id in volume_ids: - vol = db.volume_get(None, volume_id) - vol.delete() + self.volume.delete_volume(self.context, volume_id) def test_multi_node(self): # TODO(termie): Figure out how to test with two nodes, From f1c8df8d6fceebf2ba34acbc8e476bf0ecc0c94e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 00:55:19 -0700 Subject: [PATCH 058/113] instance runs --- bin/nova-compute | 2 +- bin/nova-network | 3 +- bin/nova-volume | 2 +- nova/endpoint/cloud.py | 84 +++++++++++++++++++++++------------------- 4 files changed, 50 insertions(+), 41 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index ed9a5556..cf9de9bb 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -29,4 +29,4 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.ComputeService.create() # pylint: disable-msg=C0103 + application = service.ComputeService.create() # pylint: disable=C0103 diff --git a/bin/nova-network b/bin/nova-network index 5753aafb..6434b6ec 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -33,5 +33,4 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - # pylint: disable-msg=C0103 - application = service.type_to_class(FLAGS.network_type).create() + application = service.NetworkService.create() # pylint: disable-msg=C0103 diff --git a/bin/nova-volume b/bin/nova-volume index 8ef006eb..25b5871a 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -29,4 +29,4 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.VolumeService.create() # pylint: disable-msg=C0103 + application = service.VolumeService.create() # pylint: disable-msg=C0103 diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8ba10a5b..0f3ecb3b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -233,7 +233,8 @@ class CloudController(object): return rpc.call('%s.%s' % (FLAGS.compute_topic, instance_ref['node_name']), {"method": "get_console_output", - "args": {"instance_id": instance_ref['id']}}) + "args": {"context": None, + "instance_id": instance_ref['id']}}) @rbac.allow('projectmanager', 'sysadmin') def describe_volumes(self, context, **kwargs): @@ -300,9 +301,10 @@ class CloudController(object): host = db.instance_get_host(context, instance_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "attach_volume", - "args": {"volume_id": volume_ref['id'], - "instance_id": instance_ref['id'], - "mountpoint": device}}) + "args": {"context": None, + "volume_id": volume_ref['id'], + "instance_id": instance_ref['id'], + "mountpoint": device}}) return defer.succeed({'attachTime': volume_ref['attach_time'], 'device': volume_ref['mountpoint'], 'instanceId': instance_ref['id_str'], @@ -324,8 +326,9 @@ class CloudController(object): host = db.instance_get_host(context, instance_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "detach_volume", - "args": {"instance_id": instance_ref['id'], - "volume_id": volume_ref['id']}}) + "args": {"context": None, + "instance_id": instance_ref['id'], + "volume_id": volume_ref['id']}}) except exception.NotFound: # If the instance doesn't exist anymore, # then we need to call detach blind @@ -437,7 +440,8 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) public_ip = yield rpc.call(network_topic, {"method": "allocate_floating_ip", - "args": {"project_id": context.project.id}}) + "args": {"context": None, + "project_id": context.project.id}}) defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @@ -448,7 +452,8 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "deallocate_floating_ip", - "args": {"floating_ip": floating_ip_ref['str_id']}}) + "args": {"context": None, + "floating_ip": floating_ip_ref['str_id']}}) defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') @@ -460,7 +465,8 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "associate_floating_ip", - "args": {"floating_ip": floating_ip_ref['str_id'], + "args": {"context": None, + "floating_ip": floating_ip_ref['str_id'], "fixed_ip": fixed_ip_ref['str_id'], "instance_id": instance_ref['id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @@ -472,7 +478,8 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "disassociate_floating_ip", - "args": {"floating_ip": floating_ip_ref['str_id']}}) + "args": {"context": None, + "floating_ip": floating_ip_ref['str_id']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks @@ -483,7 +490,8 @@ class CloudController(object): if not host: host = yield rpc.call(FLAGS.network_topic, {"method": "set_network_host", - "args": {"project_id": context.project.id}}) + "args": {"context": None, + "project_id": context.project.id}}) defer.returnValue(db.queue_get_for(context, FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') @@ -568,7 +576,7 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') - # @defer.inlineCallbacks + @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") # network_topic = yield self._get_network_topic(context) @@ -582,36 +590,37 @@ class CloudController(object): continue # FIXME(ja): where should network deallocate occur? - # floating_ip = network_model.get_public_ip_for_instance(i) - # if floating_ip: - # logging.debug("Disassociating address %s" % floating_ip) - # # NOTE(vish): Right now we don't really care if the ip is - # # disassociated. We may need to worry about - # # checking this later. Perhaps in the scheduler? - # rpc.cast(network_topic, - # {"method": "disassociate_floating_ip", - # "args": {"floating_ip": floating_ip}}) - # - # fixed_ip = instance.get('private_dns_name', None) - # if fixed_ip: - # logging.debug("Deallocating address %s" % fixed_ip) - # # NOTE(vish): Right now we don't really care if the ip is - # # actually removed. We may need to worry about - # # checking this later. Perhaps in the scheduler? - # rpc.cast(network_topic, - # {"method": "deallocate_fixed_ip", - # "args": {"fixed_ip": fixed_ip}}) + address = db.instance_get_floating_address(context, + instance_ref['id']) + if address: + logging.debug("Disassociating address %s" % address) + # NOTE(vish): Right now we don't really care if the ip is + # disassociated. We may need to worry about + # checking this later. Perhaps in the scheduler? + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, + {"method": "disassociate_floating_ip", + "args": {"context": None, + "address": address}}) + + address = db.instance_get_fixed_address(context, + instance_ref['id']) + if address: + logging.debug("Deallocating address %s" % address) + # NOTE(vish): Currently, nothing needs to be done on the + # network node until release. If this changes, + # we will need to cast here. + db.fixed_ip_deallocate(context, address) host = db.instance_get_host(context, instance_ref['id']) - if host is not None: - # NOTE(joshua?): It's also internal default + if host: rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "terminate_instance", - "args": {"instance_id": instance_ref['id']}}) + "args": {"context": None, + "instance_id": instance_ref['id']}}) else: db.instance_destroy(context, instance_ref['id']) - # defer.returnValue(True) - return True + defer.returnValue(True) @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): @@ -621,7 +630,8 @@ class CloudController(object): host = db.instance_get_host(context, instance_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "reboot_instance", - "args": {"instance_id": instance_ref['id']}}) + "args": {"context": None, + "instance_id": instance_ref['id']}}) return defer.succeed(True) @rbac.allow('projectmanager', 'sysadmin') From 1a49b7d40380dfd9c162d55c685118144ce3938f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 09:03:43 -0700 Subject: [PATCH 059/113] ip addresses work now --- bin/nova-dhcpbridge | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 52ec2d49..a794db27 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -36,6 +36,7 @@ from nova import rpc from nova import utils from nova import datastore # for redis_db flag from nova.auth import manager # for auth flags +from nova.network import linux_net from nova.network import manager # for network flags FLAGS = flags.FLAGS @@ -74,8 +75,7 @@ def del_lease(_mac, ip_address, _hostname, _interface): def init_leases(interface): """Get the list of hosts for an interface.""" network_ref = db.network_get_by_bridge(None, interface) - network_manager = utils.import_object(FLAGS.network_manager) - return network_manager.driver.get_dhcp_hosts(None, network_ref['id']) + return linux_net.get_dhcp_hosts(None, network_ref['id']) def main(): From eb4c52fc65b9cbf73770e0508e6f2eae8518f58c Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Mon, 30 Aug 2010 10:51:54 -0700 Subject: [PATCH 060/113] Making tests pass --- nova/tests/network_unittest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index e0de04be..d487c2e4 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -105,7 +105,6 @@ class NetworkTestCase(test.TrialTestCase): db.fixed_ip_deallocate(None, fix_addr) def test_allocate_deallocate_fixed_ip(self): - import pdb; pdb.set_trace() """Makes sure that we can allocate and deallocate a fixed ip""" address = self._create_address(0) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) From d1ea445f351fae39138204edf1bc8323e302bec6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 15:11:46 -0700 Subject: [PATCH 061/113] all tests pass again --- nova/endpoint/cloud.py | 4 ++-- nova/tests/compute_unittest.py | 30 +++++++++++++++++------------- nova/tests/network_unittest.py | 7 +++---- nova/tests/volume_unittest.py | 7 ++++--- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 0f3ecb3b..4f7f1c60 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -94,7 +94,7 @@ class CloudController(object): return result def get_metadata(self, ipaddress): - i = db.instance_get_by_address(ipaddress) + i = db.fixed_ip_get_instance(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -421,7 +421,7 @@ class CloudController(object): context.project.id) for floating_ip_ref in iterator: address = floating_ip_ref['id_str'] - instance_ref = db.instance_get_by_address(address) + instance_ref = db.floating_ip_get_instance(address) address_rv = { 'public_ip': address, 'instance_id': instance_ref['id_str'] diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 28e51f38..a8d644c8 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -40,7 +40,7 @@ class InstanceXmlTestCase(test.TrialTestCase): # instance_id = 'foo' # first_node = node.Node() - # inst = yield first_node.run_instance(instance_id) + # inst = yield first_node.run_instance(self.context, instance_id) # # # force the state so that we can verify that it changes # inst._s['state'] = node.Instance.NOSTATE @@ -50,7 +50,7 @@ class InstanceXmlTestCase(test.TrialTestCase): # second_node = node.Node() # new_inst = node.Instance.fromXml(second_node._conn, pool=second_node._pool, xml=xml) # self.assertEqual(new_inst.state, node.Instance.RUNNING) - # rv = yield first_node.terminate_instance(instance_id) + # rv = yield first_node.terminate_instance(self.context, instance_id) class ComputeConnectionTestCase(test.TrialTestCase): @@ -63,6 +63,7 @@ class ComputeConnectionTestCase(test.TrialTestCase): self.manager = manager.AuthManager() user = self.manager.create_user('fake', 'fake', 'fake') project = self.manager.create_project('fake', 'fake', 'fake') + self.context = None def tearDown(self): self.manager.delete_user('fake') @@ -84,13 +85,13 @@ class ComputeConnectionTestCase(test.TrialTestCase): def test_run_describe_terminate(self): instance_id = self._create_instance() - yield self.compute.run_instance(instance_id) + yield self.compute.run_instance(self.context, instance_id) instances = db.instance_get_all(None) logging.info("Running instances: %s", instances) self.assertEqual(len(instances), 1) - yield self.compute.terminate_instance(instance_id) + yield self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(None) logging.info("After terminating instances: %s", instances) @@ -99,22 +100,25 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_reboot(self): instance_id = self._create_instance() - yield self.compute.run_instance(instance_id) - yield self.compute.reboot_instance(instance_id) - yield self.compute.terminate_instance(instance_id) + yield self.compute.run_instance(self.context, instance_id) + yield self.compute.reboot_instance(self.context, instance_id) + yield self.compute.terminate_instance(self.context, instance_id) @defer.inlineCallbacks def test_console_output(self): instance_id = self._create_instance() - rv = yield self.compute.run_instance(instance_id) + rv = yield self.compute.run_instance(self.context, instance_id) - console = yield self.compute.get_console_output(instance_id) + console = yield self.compute.get_console_output(self.context, + instance_id) self.assert_(console) - rv = yield self.compute.terminate_instance(instance_id) + rv = yield self.compute.terminate_instance(self.context, instance_id) @defer.inlineCallbacks def test_run_instance_existing(self): instance_id = self._create_instance() - yield self.compute.run_instance(instance_id) - self.assertFailure(self.compute.run_instance(instance_id), exception.Error) - yield self.compute.terminate_instance(instance_id) + yield self.compute.run_instance(self.context, instance_id) + self.assertFailure(self.compute.run_instance(self.context, + instance_id), + exception.Error) + yield self.compute.terminate_instance(self.context, instance_id) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index e3fe01fa..b479f2fa 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -253,12 +253,11 @@ class NetworkTestCase(test.TrialTestCase): def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" - fixed_ip = db.fixed_ip_get_by_address(None, address) project_net = db.project_get_network(None, project_id) + network = db.fixed_ip_get_network(None, address) + instance = db.fixed_ip_get_instance(None, address) # instance exists until release - logging.debug('fixed_ip.instance: %s', fixed_ip.instance) - logging.debug('project_net: %s', project_net) - return fixed_ip.instance is not None and fixed_ip.network == project_net + return instance is not None and network['id'] == project_net['id'] def binpath(script): diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 4504276e..6573e987 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -117,6 +117,7 @@ class VolumeTestCase(test.TrialTestCase): else: rv = yield self.compute.detach_volume(instance_id, volume_id) + vol = db.volume_get(None, volume_id) self.assertEqual(vol['status'], "available") rv = self.volume.delete_volume(self.context, volume_id) @@ -134,9 +135,9 @@ class VolumeTestCase(test.TrialTestCase): volume_ids = [] def _check(volume_id): volume_ids.append(volume_id) - vol = db.volume_get(None, volume_id) - shelf_blade = '%s.%s' % (vol.export_device.shelf_id, - vol.export_device.blade_id) + (shelf_id, blade_id) = db.volume_get_shelf_and_blade(None, + volume_id) + shelf_blade = '%s.%s' % (shelf_id, blade_id) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) From 67a6444b3448828e8c6fce2e8b5e4a0c0327d7e1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 17:58:02 -0700 Subject: [PATCH 062/113] more pep8 --- bin/nova-dhcpbridge | 7 ++++--- bin/nova-manage | 4 ++-- bin/nova-objectstore | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index a794db27..c416d07a 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -34,13 +34,14 @@ from nova import db from nova import flags from nova import rpc from nova import utils -from nova import datastore # for redis_db flag -from nova.auth import manager # for auth flags +from nova import datastore # for redis_db flag +from nova.auth import manager # for auth flags from nova.network import linux_net -from nova.network import manager # for network flags +from nova.network import manager # for network flags FLAGS = flags.FLAGS + def add_lease(_mac, ip_address, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: diff --git a/bin/nova-manage b/bin/nova-manage index 145294d3..7f20531d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -56,8 +56,8 @@ class VpnCommands(object): vpn = self._vpn_for(project.id) if vpn: command = "ping -c1 -w1 %s > /dev/null; echo $?" - out, _err = utils.execute( command % vpn['private_dns_name'], - check_exit_code=False) + out, _err = utils.execute(command % vpn['private_dns_name'], + check_exit_code=False) if out.strip() == '0': net = 'up' else: diff --git a/bin/nova-objectstore b/bin/nova-objectstore index afcf13e2..7cb718b6 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -35,4 +35,4 @@ if __name__ == '__main__': if __name__ == '__builtin__': utils.default_flagfile() - application = handler.get_application() # pylint: disable-msg=C0103 + application = handler.get_application() # pylint: disable-msg=C0103 From d20f6b7c5f7f43e00276c01cdc39c3bc8cb26acb Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 18:10:52 -0700 Subject: [PATCH 063/113] more pep8 --- nova/tests/compute_unittest.py | 3 ++- nova/tests/network_unittest.py | 6 ++---- nova/tests/service_unittest.py | 27 ++++++++++++--------------- nova/tests/volume_unittest.py | 3 +-- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index a8d644c8..0166dc4b 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -48,7 +48,8 @@ class InstanceXmlTestCase(test.TrialTestCase): # self.assert_(ElementTree.parse(StringIO.StringIO(xml))) # # second_node = node.Node() - # new_inst = node.Instance.fromXml(second_node._conn, pool=second_node._pool, xml=xml) + # new_inst = node.Instance.fromXml(second_node._conn, + # pool=second_node._pool, xml=xml) # self.assertEqual(new_inst.state, node.Instance.RUNNING) # rv = yield first_node.terminate_instance(self.context, instance_id) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index b284e4e5..15ec8dbf 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -140,7 +140,8 @@ class NetworkTestCase(test.TrialTestCase): db.fixed_ip_deallocate(None, address2) release_ip(address2) - self.assertFalse(is_allocated_in_project(address2, self.projects[1].id)) + self.assertFalse(is_allocated_in_project(address2, + self.projects[1].id)) def test_subnet_edge(self): """Makes sure that private ips don't overlap""" @@ -190,7 +191,6 @@ class NetworkTestCase(test.TrialTestCase): for project in projects: self.manager.delete_project(project) - def test_ips_are_reused(self): """Makes sure that ip addresses that are deallocated get reused""" address = self._create_address(0) @@ -224,8 +224,6 @@ class NetworkTestCase(test.TrialTestCase): """Test for a NoMoreAddresses exception when all fixed ips are used. """ network = db.project_get_network(None, self.projects[0].id) - - num_available_ips = db.network_count_available_ips(None, network['id']) addresses = [] diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e13fe62d..902f9bab 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -20,10 +20,7 @@ Unit Tests for remote procedure calls using queue """ -import logging - import mox -from twisted.internet import defer from nova import exception from nova import flags @@ -33,33 +30,37 @@ from nova import service from nova import manager FLAGS = flags.FLAGS - flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager", "Manager for testing") + class FakeManager(manager.Manager): """Fake manager for tests""" pass + class ServiceTestCase(test.BaseTestCase): """Test cases for rpc""" + def setUp(self): # pylint: disable=C0103 super(ServiceTestCase, self).setUp() self.mox.StubOutWithMock(service, 'db') def test_create(self): - self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) + self.mox.StubOutWithMock(rpc, + 'AdapterConsumer', + use_mock_anything=True) self.mox.StubOutWithMock( service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic='fake', - proxy=mox.IsA(service.Service) - ).AndReturn(rpc.AdapterConsumer) + proxy=mox.IsA(service.Service)).AndReturn( + rpc.AdapterConsumer) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic='fake.%s' % FLAGS.node_name, - proxy=mox.IsA(service.Service) - ).AndReturn(rpc.AdapterConsumer) + proxy=mox.IsA(service.Service)).AndReturn( + rpc.AdapterConsumer) # Stub out looping call a bit needlessly since we don't have an easy # way to cancel it (yet) when the tests finishes @@ -80,7 +81,6 @@ class ServiceTestCase(test.BaseTestCase): # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - def test_report_state(self): node_name = 'foo' binary = 'bar' @@ -99,7 +99,6 @@ class ServiceTestCase(test.BaseTestCase): s = service.Service() rv = yield s.report_state(node_name, binary) - def test_report_state_no_daemon(self): node_name = 'foo' binary = 'bar' @@ -115,7 +114,8 @@ class ServiceTestCase(test.BaseTestCase): service.db.daemon_get_by_args(None, node_name, binary).AndRaise(exception.NotFound()) - service.db.daemon_create(None, daemon_create).AndReturn(daemon_ref['id']) + service.db.daemon_create(None, + daemon_create).AndReturn(daemon_ref['id']) service.db.daemon_get(None, daemon_ref['id']).AndReturn(daemon_ref) service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) @@ -124,7 +124,6 @@ class ServiceTestCase(test.BaseTestCase): s = service.Service() rv = yield s.report_state(node_name, binary) - def test_report_state_newly_disconnected(self): node_name = 'foo' binary = 'bar' @@ -144,7 +143,6 @@ class ServiceTestCase(test.BaseTestCase): self.assert_(s.model_disconnected) - def test_report_state_newly_connected(self): node_name = 'foo' binary = 'bar' @@ -166,4 +164,3 @@ class ServiceTestCase(test.BaseTestCase): rv = yield s.report_state(node_name, binary) self.assert_(not s.model_disconnected) - diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 6573e987..f42d0ac8 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -26,7 +26,6 @@ from nova import flags from nova import test from nova import utils - FLAGS = flags.FLAGS @@ -40,7 +39,6 @@ class VolumeTestCase(test.TrialTestCase): self.volume = utils.import_object(FLAGS.volume_manager) self.context = None - def _create_volume(self, size='0'): vol = {} vol['size'] = '0' @@ -133,6 +131,7 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' shelf_blades = [] volume_ids = [] + def _check(volume_id): volume_ids.append(volume_id) (shelf_id, blade_id) = db.volume_get_shelf_and_blade(None, From 7577db5c9acaceabb50db8741f47a8702487baa3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 20:42:31 -0700 Subject: [PATCH 064/113] more cleanup and pylint fixes --- nova/auth/manager.py | 2 +- nova/tests/network_unittest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 62ec3f4e..d5fbec7c 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -574,7 +574,7 @@ class AuthManager(object): if not network_ref['vpn_public_port']: raise exception.NotFound('project network data has not been set') - return (network_ref['vpn_public_ip_str'], + return (network_ref['vpn_public_address'], network_ref['vpn_public_port']) def delete_project(self, project, context=None): diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 15ec8dbf..fccfc23f 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -28,7 +28,6 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager -from nova.network import service FLAGS = flags.FLAGS From 175b4a332883f3fb944ef2d8d68a9c54b1f04773 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 30 Aug 2010 22:13:22 -0700 Subject: [PATCH 065/113] pylint cleanup of tests --- nova/tests/compute_unittest.py | 61 +++++++++++++--------------------- nova/tests/network_unittest.py | 3 +- nova/tests/volume_unittest.py | 45 ++++++++++++++----------- 3 files changed, 50 insertions(+), 59 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 0166dc4b..867b572f 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -15,11 +15,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +""" +Tests For Compute +""" import logging -import time + from twisted.internet import defer -from xml.etree import ElementTree from nova import db from nova import exception @@ -32,58 +33,39 @@ from nova.auth import manager FLAGS = flags.FLAGS -class InstanceXmlTestCase(test.TrialTestCase): - # @defer.inlineCallbacks - def test_serialization(self): - # TODO: Reimplement this, it doesn't make sense in redis-land - return - - # instance_id = 'foo' - # first_node = node.Node() - # inst = yield first_node.run_instance(self.context, instance_id) - # - # # force the state so that we can verify that it changes - # inst._s['state'] = node.Instance.NOSTATE - # xml = inst.toXml() - # self.assert_(ElementTree.parse(StringIO.StringIO(xml))) - # - # second_node = node.Node() - # new_inst = node.Instance.fromXml(second_node._conn, - # pool=second_node._pool, xml=xml) - # self.assertEqual(new_inst.state, node.Instance.RUNNING) - # rv = yield first_node.terminate_instance(self.context, instance_id) - - -class ComputeConnectionTestCase(test.TrialTestCase): - def setUp(self): +class ComputeTestCase(test.TrialTestCase): + """Test case for compute""" + def setUp(self): # pylint: disable-msg=C0103 logging.getLogger().setLevel(logging.DEBUG) - super(ComputeConnectionTestCase, self).setUp() + super(ComputeTestCase, self).setUp() self.flags(connection_type='fake', fake_storage=True) self.compute = utils.import_object(FLAGS.compute_manager) self.manager = manager.AuthManager() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') + self.user = self.manager.create_user('fake', 'fake', 'fake') + self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = None - def tearDown(self): - self.manager.delete_user('fake') - self.manager.delete_project('fake') + def tearDown(self): # pylint: disable-msg=C0103 + self.manager.delete_user(self.user) + self.manager.delete_project(self.project) def _create_instance(self): + """Create a test instance""" inst = {} inst['image_id'] = 'ami-test' inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' - inst['user_id'] = 'fake' - inst['project_id'] = 'fake' + inst['user_id'] = self.user.id + inst['project_id'] = self.project.id inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 return db.instance_create(None, inst) @defer.inlineCallbacks - def test_run_describe_terminate(self): + def test_run_terminate(self): + """Make sure it is possible to run and terminate instance""" instance_id = self._create_instance() yield self.compute.run_instance(self.context, instance_id) @@ -100,6 +82,7 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_reboot(self): + """Ensure instance can be rebooted""" instance_id = self._create_instance() yield self.compute.run_instance(self.context, instance_id) yield self.compute.reboot_instance(self.context, instance_id) @@ -107,16 +90,18 @@ class ComputeConnectionTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_console_output(self): + """Make sure we can get console output from instance""" instance_id = self._create_instance() - rv = yield self.compute.run_instance(self.context, instance_id) + yield self.compute.run_instance(self.context, instance_id) console = yield self.compute.get_console_output(self.context, instance_id) self.assert_(console) - rv = yield self.compute.terminate_instance(self.context, instance_id) + yield self.compute.terminate_instance(self.context, instance_id) @defer.inlineCallbacks def test_run_instance_existing(self): + """Ensure failure when running an instance that already exists""" instance_id = self._create_instance() yield self.compute.run_instance(self.context, instance_id) self.assertFailure(self.compute.run_instance(self.context, diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index fccfc23f..7cd20dfc 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -75,6 +75,7 @@ class NetworkTestCase(test.TrialTestCase): self.manager.delete_user(self.user) def _create_address(self, project_num, instance_id=None): + """Create an address in given project num""" net = db.project_get_network(None, self.projects[project_num].id) address = db.fixed_ip_allocate(None, net['id']) if instance_id is None: @@ -147,7 +148,6 @@ class NetworkTestCase(test.TrialTestCase): first = self._create_address(0) lease_ip(first) for i in range(1, 5): - project_id = self.projects[i].id address = self._create_address(i) address2 = self._create_address(i) address3 = self._create_address(i) @@ -227,7 +227,6 @@ class NetworkTestCase(test.TrialTestCase): network['id']) addresses = [] for i in range(num_available_ips): - project_id = self.projects[0].id address = self._create_address(0) addresses.append(address) lease_ip(address) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index f42d0ac8..0df0c20d 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -15,7 +15,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +""" +Tests for Volume Code +""" import logging from twisted.internet import defer @@ -30,7 +32,8 @@ FLAGS = flags.FLAGS class VolumeTestCase(test.TrialTestCase): - def setUp(self): + """Test Case for volumes""" + def setUp(self): # pylint: disable-msg=C0103 logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() self.compute = utils.import_object(FLAGS.compute_manager) @@ -39,9 +42,11 @@ class VolumeTestCase(test.TrialTestCase): self.volume = utils.import_object(FLAGS.volume_manager) self.context = None - def _create_volume(self, size='0'): + @staticmethod + def _create_volume(size='0'): + """Create a volume object""" vol = {} - vol['size'] = '0' + vol['size'] = size vol['user_id'] = 'fake' vol['project_id'] = 'fake' vol['availability_zone'] = FLAGS.storage_availability_zone @@ -50,7 +55,8 @@ class VolumeTestCase(test.TrialTestCase): return db.volume_create(None, vol)['id'] @defer.inlineCallbacks - def test_run_create_volume(self): + def test_create_delete_volume(self): + """Test volume can be created and deleted""" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(None, volume_id).id) @@ -63,6 +69,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_big_volume(self): + """Ensure failure if a too large of a volume is requested""" # FIXME(vish): validation needs to move into the data layer in # volume_create defer.returnValue(True) @@ -75,9 +82,10 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): + """Ensure that NoMoreBlades is raised when we run out of volumes""" vols = [] total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf - for i in xrange(total_slots): + for _index in xrange(total_slots): volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) vols.append(volume_id) @@ -91,7 +99,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_attach_detach_volume(self): - # Create one volume and one compute to test with + """Make sure volume can be attached and detached from instance""" instance_id = "storage-test" mountpoint = "/dev/sdf" volume_id = self._create_volume() @@ -99,9 +107,9 @@ class VolumeTestCase(test.TrialTestCase): if FLAGS.fake_tests: db.volume_attached(None, volume_id, instance_id, mountpoint) else: - rv = yield self.compute.attach_volume(instance_id, - volume_id, - mountpoint) + yield self.compute.attach_volume(instance_id, + volume_id, + mountpoint) vol = db.volume_get(None, volume_id) self.assertEqual(vol['status'], "in-use") self.assertEqual(vol['attach_status'], "attached") @@ -113,12 +121,12 @@ class VolumeTestCase(test.TrialTestCase): if FLAGS.fake_tests: db.volume_detached(None, volume_id) else: - rv = yield self.compute.detach_volume(instance_id, - volume_id) + yield self.compute.detach_volume(instance_id, + volume_id) vol = db.volume_get(None, volume_id) self.assertEqual(vol['status'], "available") - rv = self.volume.delete_volume(self.context, volume_id) + yield self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.Error, db.volume_get, None, @@ -126,23 +134,22 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_concurrent_volumes_get_different_blades(self): - vol_size = "5" - user_id = "fake" - project_id = 'fake' - shelf_blades = [] + """Ensure multiple concurrent volumes get different blades""" volume_ids = [] + shelf_blades = [] def _check(volume_id): + """Make sure blades aren't duplicated""" volume_ids.append(volume_id) (shelf_id, blade_id) = db.volume_get_shelf_and_blade(None, volume_id) shelf_blade = '%s.%s' % (shelf_id, blade_id) self.assert_(shelf_blade not in shelf_blades) shelf_blades.append(shelf_blade) - logging.debug("got %s" % shelf_blade) + logging.debug("Blade %s allocated", shelf_blade) deferreds = [] total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf - for i in range(total_slots): + for _index in xrange(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(self.context, volume_id) d.addCallback(_check) From 63eaec4e1a6e1d5aff8d602e44d8b3a33985632d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 31 Aug 2010 21:17:48 +0200 Subject: [PATCH 067/113] Better log formatter for Nova. It's just like gnuchangelog, but logs the author rather than the committer. --- bzrplugins/novalog/__init__.py | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 bzrplugins/novalog/__init__.py diff --git a/bzrplugins/novalog/__init__.py b/bzrplugins/novalog/__init__.py new file mode 100644 index 00000000..e16b2e00 --- /dev/null +++ b/bzrplugins/novalog/__init__.py @@ -0,0 +1,59 @@ +# Copyright 2010 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Log format for Nova's changelog.""" + +import bzrlib.log +from bzrlib.osutils import format_date + +# +# This is mostly stolen from bzrlib.log.GnuChangelogLogFormatter +# The difference is that it logs the author rather than the committer +# which for Nova always is Tarmac. +# +class NovaLogFormat(bzrlib.log.GnuChangelogLogFormatter): + preferred_levels = 1 + def log_revision(self, revision): + """Log a revision, either merged or not.""" + to_file = self.to_file + + date_str = format_date(revision.rev.timestamp, + revision.rev.timezone or 0, + self.show_timezone, + date_fmt='%Y-%m-%d', + show_offset=False) + + authors = revision.rev.get_apparent_authors() + to_file.write('%s %s\n\n' % (date_str, ", ".join(authors))) + + if revision.delta is not None and revision.delta.has_changed(): + for c in revision.delta.added + revision.delta.removed + revision.delta.modified: + path, = c[:1] + to_file.write('\t* %s:\n' % (path,)) + for c in revision.delta.renamed: + oldpath,newpath = c[:2] + # For renamed files, show both the old and the new path + to_file.write('\t* %s:\n\t* %s:\n' % (oldpath,newpath)) + to_file.write('\n') + + if not revision.rev.message: + to_file.write('\tNo commit message\n') + else: + message = revision.rev.message.rstrip('\r\n') + for l in message.split('\n'): + to_file.write('\t%s\n' % (l.lstrip(),)) + to_file.write('\n') + +bzrlib.log.register_formatter('novalog', NovaLogFormat) + From e63eb090a418de458825db077dbe56343361393d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 31 Aug 2010 16:48:41 -0700 Subject: [PATCH 068/113] rename node_name to host --- nova/endpoint/cloud.py | 6 +++--- nova/flags.py | 2 +- nova/tests/model_unittest.py | 6 +++--- nova/tests/network_unittest.py | 2 +- nova/tests/service_unittest.py | 36 +++++++++++++++++----------------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8e459c93..c34eb5da 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -230,7 +230,7 @@ class CloudController(object): # instance_id is passed in as a list of instances instance_ref = db.instance_get_by_str(context, instance_id[0]) return rpc.call('%s.%s' % (FLAGS.compute_topic, - instance_ref['node_name']), + instance_ref['host']), {"method": "get_console_output", "args": {"context": None, "instance_id": instance_ref['id']}}) @@ -257,7 +257,7 @@ class CloudController(object): v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], volume['user_id'], - 'node_name', + 'host', volume['instance_id'], volume['mountpoint']) if volume['attach_status'] == 'attached': @@ -391,7 +391,7 @@ class CloudController(object): if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], instance.project_id, - 'node_name') # FIXME + 'host') # FIXME i['product_codes_set'] = self._convert_to_set([], 'product_codes') i['instance_type'] = instance.instance_type i['launch_time'] = instance.created_at diff --git a/nova/flags.py b/nova/flags.py index a9917983..ebbfe3ff 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -206,7 +206,7 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') # UNUSED DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') -DEFINE_string('node_name', socket.gethostname(), +DEFINE_string('host', socket.gethostname(), 'name of this node') DEFINE_string('sql_connection', diff --git a/nova/tests/model_unittest.py b/nova/tests/model_unittest.py index dc2441c2..130516c6 100644 --- a/nova/tests/model_unittest.py +++ b/nova/tests/model_unittest.py @@ -108,14 +108,14 @@ class ModelTestCase(test.TrialTestCase): self.assertEqual(x.identifier, 'i-test') def test_instance_associates_node(self): - """create, then check that it is listed for the node_name""" + """create, then check that it is listed for the host""" instance = self.create_instance() found = False - for x in model.InstanceDirectory().by_node(FLAGS.node_name): + for x in model.InstanceDirectory().by_node(FLAGS.host): if x.identifier == 'i-test': found = True self.assertFalse(found) - instance['node_name'] = 'test_node' + instance['host'] = 'test_node' instance.save() for x in model.InstanceDirectory().by_node('test_node'): if x.identifier == 'i-test': diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 7cd20dfc..f3124c1b 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -91,7 +91,7 @@ class NetworkTestCase(test.TrialTestCase): try: db.floating_ip_get_by_address(None, ip_str) except exception.NotFound: - db.floating_ip_create(None, ip_str, FLAGS.node_name) + db.floating_ip_create(None, ip_str, FLAGS.host) float_addr = self.network.allocate_floating_ip(self.context, self.projects[0].id) fix_addr = self._create_address(0) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 902f9bab..318abe64 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -58,7 +58,7 @@ class ServiceTestCase(test.BaseTestCase): rpc.AdapterConsumer) rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='fake.%s' % FLAGS.node_name, + topic='fake.%s' % FLAGS.host, proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) @@ -82,37 +82,37 @@ class ServiceTestCase(test.BaseTestCase): # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage def test_report_state(self): - node_name = 'foo' + host = 'foo' binary = 'bar' - daemon_ref = {'node_name': node_name, + daemon_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, - node_name, + host, binary).AndReturn(daemon_ref) service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() s = service.Service() - rv = yield s.report_state(node_name, binary) + rv = yield s.report_state(host, binary) def test_report_state_no_daemon(self): - node_name = 'foo' + host = 'foo' binary = 'bar' - daemon_create = {'node_name': node_name, + daemon_create = {'host': host, 'binary': binary, 'report_count': 0} - daemon_ref = {'node_name': node_name, + daemon_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, - node_name, + host, binary).AndRaise(exception.NotFound()) service.db.daemon_create(None, daemon_create).AndReturn(daemon_ref['id']) @@ -122,38 +122,38 @@ class ServiceTestCase(test.BaseTestCase): self.mox.ReplayAll() s = service.Service() - rv = yield s.report_state(node_name, binary) + rv = yield s.report_state(host, binary) def test_report_state_newly_disconnected(self): - node_name = 'foo' + host = 'foo' binary = 'bar' - daemon_ref = {'node_name': node_name, + daemon_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, - node_name, + host, binary).AndRaise(Exception()) self.mox.ReplayAll() s = service.Service() - rv = yield s.report_state(node_name, binary) + rv = yield s.report_state(host, binary) self.assert_(s.model_disconnected) def test_report_state_newly_connected(self): - node_name = 'foo' + host = 'foo' binary = 'bar' - daemon_ref = {'node_name': node_name, + daemon_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') service.db.daemon_get_by_args(None, - node_name, + host, binary).AndReturn(daemon_ref) service.db.daemon_update(None, daemon_ref['id'], mox.ContainsKeyValue('report_count', 1)) @@ -161,6 +161,6 @@ class ServiceTestCase(test.BaseTestCase): self.mox.ReplayAll() s = service.Service() s.model_disconnected = True - rv = yield s.report_state(node_name, binary) + rv = yield s.report_state(host, binary) self.assert_(not s.model_disconnected) From 16b82b4558a374443f2e86fbbf5c165ba1565074 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 11:25:10 -0700 Subject: [PATCH 070/113] updated models a bit and removed service classes --- bin/nova-compute | 4 ++-- bin/nova-network | 8 ++------ bin/nova-volume | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index cf9de9bb..cc4c9e2f 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -21,12 +21,12 @@ Twistd daemon for the nova compute nodes. """ +from nova import service from nova import twistd -from nova.compute import service if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.ComputeService.create() # pylint: disable=C0103 + application = service.Service.create() # pylint: disable=C0103 diff --git a/bin/nova-network b/bin/nova-network index 6434b6ec..040b35e0 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -21,16 +21,12 @@ Twistd daemon for the nova network nodes. """ -from nova import flags +from nova import service from nova import twistd -from nova.network import service - -FLAGS = flags.FLAGS - if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.NetworkService.create() # pylint: disable-msg=C0103 + application = service.Service.create() # pylint: disable-msg=C0103 diff --git a/bin/nova-volume b/bin/nova-volume index 25b5871a..fac4b5d0 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -21,12 +21,12 @@ Twistd daemon for the nova volume nodes. """ +from nova import service from nova import twistd -from nova.volume import service if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.VolumeService.create() # pylint: disable-msg=C0103 + application = service.Service.create() # pylint: disable-msg=C0103 From 20c3252cf69810c0e90f57707718ef169ae26f51 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 14:13:22 -0700 Subject: [PATCH 071/113] renamed daemon to service and update db on create and destroy --- nova/endpoint/images.py | 2 +- nova/server.py | 30 ++++++++++++++-------------- nova/tests/service_unittest.py | 36 +++++++++++++++++----------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index f72c277a..4579cd81 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -18,7 +18,7 @@ """ Proxy AMI-related calls from the cloud controller, to the running -objectstore daemon. +objectstore service. """ import json diff --git a/nova/server.py b/nova/server.py index c6b60e09..8cc1e0ff 100644 --- a/nova/server.py +++ b/nova/server.py @@ -17,11 +17,11 @@ # under the License. """ -Base functionality for nova daemons - gradually being replaced with twistd.py. +Base functionality for nova services - gradually being replaced with twistd.py. """ -import daemon -from daemon import pidlockfile +import service +from service import pidlockfile import logging import logging.handlers import os @@ -33,14 +33,14 @@ from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_bool('daemonize', False, 'daemonize this process') -# NOTE(termie): right now I am defaulting to using syslog when we daemonize +flags.DEFINE_bool('serviceize', False, 'serviceize this process') +# NOTE(termie): right now I am defaulting to using syslog when we serviceize # it may be better to do something else -shrug- # NOTE(Devin): I think we should let each process have its own log file # and put it in /var/logs/nova/(appname).log # This makes debugging much easier and cuts down on sys log # clutter. -flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') +flags.DEFINE_bool('use_syslog', True, 'output to syslog when serviceizing') flags.DEFINE_string('logfile', None, 'log file to output to') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') @@ -50,17 +50,17 @@ flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') def stop(pidfile): """ - Stop the daemon + Stop the service """ # Get the pid from the pidfile try: pid = int(open(pidfile,'r').read().strip()) except IOError: - message = "pidfile %s does not exist. Daemon not running?\n" + message = "pidfile %s does not exist. Service not running?\n" sys.stderr.write(message % pidfile) return # not an error in a restart - # Try killing the daemon process + # Try killing the service process try: while 1: os.kill(pid, signal.SIGTERM) @@ -100,13 +100,13 @@ def serve(name, main): else: print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - daemonize(argv, name, main) + serviceize(argv, name, main) -def daemonize(args, name, main): - """Does the work of daemonizing the process""" +def serviceize(args, name, main): + """Does the work of serviceizing the process""" logging.getLogger('amqplib').setLevel(logging.WARN) - if FLAGS.daemonize: + if FLAGS.serviceize: logger = logging.getLogger() formatter = logging.Formatter( name + '(%(name)s): %(levelname)s %(message)s') @@ -129,8 +129,8 @@ def daemonize(args, name, main): else: logging.getLogger().setLevel(logging.WARNING) - with daemon.DaemonContext( - detach_process=FLAGS.daemonize, + with service.ServiceContext( + detach_process=FLAGS.serviceize, working_directory=FLAGS.working_directory, pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, acquire_timeout=1, diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 318abe64..274e74b5 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -84,40 +84,40 @@ class ServiceTestCase(test.BaseTestCase): def test_report_state(self): host = 'foo' binary = 'bar' - daemon_ref = {'host': host, + service_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') - service.db.daemon_get_by_args(None, + service.db.service_get_by_args(None, host, - binary).AndReturn(daemon_ref) - service.db.daemon_update(None, daemon_ref['id'], + binary).AndReturn(service_ref) + service.db.service_update(None, service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() s = service.Service() rv = yield s.report_state(host, binary) - def test_report_state_no_daemon(self): + def test_report_state_no_service(self): host = 'foo' binary = 'bar' - daemon_create = {'host': host, + service_create = {'host': host, 'binary': binary, 'report_count': 0} - daemon_ref = {'host': host, + service_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') - service.db.daemon_get_by_args(None, + service.db.service_get_by_args(None, host, binary).AndRaise(exception.NotFound()) - service.db.daemon_create(None, - daemon_create).AndReturn(daemon_ref['id']) - service.db.daemon_get(None, daemon_ref['id']).AndReturn(daemon_ref) - service.db.daemon_update(None, daemon_ref['id'], + service.db.service_create(None, + service_create).AndReturn(service_ref['id']) + service.db.service_get(None, service_ref['id']).AndReturn(service_ref) + service.db.service_update(None, service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() @@ -127,13 +127,13 @@ class ServiceTestCase(test.BaseTestCase): def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' - daemon_ref = {'host': host, + service_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') - service.db.daemon_get_by_args(None, + service.db.service_get_by_args(None, host, binary).AndRaise(Exception()) @@ -146,16 +146,16 @@ class ServiceTestCase(test.BaseTestCase): def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' - daemon_ref = {'host': host, + service_ref = {'host': host, 'binary': binary, 'report_count': 0, 'id': 1} service.db.__getattr__('report_state') - service.db.daemon_get_by_args(None, + service.db.service_get_by_args(None, host, - binary).AndReturn(daemon_ref) - service.db.daemon_update(None, daemon_ref['id'], + binary).AndReturn(service_ref) + service.db.service_update(None, service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() From 0845cf8ddde2477e9adb5a1aec2eca2f9d0cc6c3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 14:43:02 -0700 Subject: [PATCH 072/113] removed dangling files --- nova/datastore.old.py | 261 ------------------------------- nova/tests/model_unittest.py | 292 ----------------------------------- run_tests.py | 1 - 3 files changed, 554 deletions(-) delete mode 100644 nova/datastore.old.py delete mode 100644 nova/tests/model_unittest.py diff --git a/nova/datastore.old.py b/nova/datastore.old.py deleted file mode 100644 index 751c5eee..00000000 --- a/nova/datastore.old.py +++ /dev/null @@ -1,261 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Datastore: - -MAKE Sure that ReDIS is running, and your flags are set properly, -before trying to run this. -""" - -import logging - -from nova import exception -from nova import flags -from nova import utils - - -FLAGS = flags.FLAGS -flags.DEFINE_string('redis_host', '127.0.0.1', - 'Host that redis is running on.') -flags.DEFINE_integer('redis_port', 6379, - 'Port that redis is running on.') -flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') - - -class Redis(object): - def __init__(self): - if hasattr(self.__class__, '_instance'): - raise Exception('Attempted to instantiate singleton') - - @classmethod - def instance(cls): - if not hasattr(cls, '_instance'): - inst = redis.Redis(host=FLAGS.redis_host, - port=FLAGS.redis_port, - db=FLAGS.redis_db) - cls._instance = inst - return cls._instance - - -class ConnectionError(exception.Error): - pass - - -def absorb_connection_error(fn): - def _wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except redis.exceptions.ConnectionError, ce: - raise ConnectionError(str(ce)) - return _wrapper - - -class BasicModel(object): - """ - All Redis-backed data derives from this class. - - You MUST specify an identifier() property that returns a unique string - per instance. - - You MUST have an initializer that takes a single argument that is a value - returned by identifier() to load a new class with. - - You may want to specify a dictionary for default_state(). - - You may also specify override_type at the class left to use a key other - than __class__.__name__. - - You override save and destroy calls to automatically build and destroy - associations. - """ - - override_type = None - - @absorb_connection_error - def __init__(self): - state = Redis.instance().hgetall(self.__redis_key) - if state: - self.initial_state = state - self.state = dict(self.initial_state) - else: - self.initial_state = {} - self.state = self.default_state() - - - def default_state(self): - """You probably want to define this in your subclass""" - return {} - - @classmethod - def _redis_name(cls): - return cls.override_type or cls.__name__.lower() - - @classmethod - def lookup(cls, identifier): - rv = cls(identifier) - if rv.is_new_record(): - return None - else: - return rv - - @classmethod - @absorb_connection_error - def all(cls): - """yields all objects in the store""" - redis_set = cls._redis_set_name(cls.__name__) - for identifier in Redis.instance().smembers(redis_set): - yield cls(identifier) - - @classmethod - def associated_to(cls, foreign_type, foreign_id): - for identifier in cls.associated_keys(foreign_type, foreign_id): - yield cls(identifier) - - @classmethod - @absorb_connection_error - def associated_keys(cls, foreign_type, foreign_id): - redis_set = cls._redis_association_name(foreign_type, foreign_id) - return Redis.instance().smembers(redis_set) or [] - - @classmethod - def _redis_set_name(cls, kls_name): - # stupidly pluralize (for compatiblity with previous codebase) - return kls_name.lower() + "s" - - @classmethod - def _redis_association_name(cls, foreign_type, foreign_id): - return cls._redis_set_name("%s:%s:%s" % - (foreign_type, foreign_id, cls._redis_name())) - - @property - def identifier(self): - """You DEFINITELY want to define this in your subclass""" - raise NotImplementedError("Your subclass should define identifier") - - @property - def __redis_key(self): - return '%s:%s' % (self._redis_name(), self.identifier) - - def __repr__(self): - return "<%s:%s>" % (self.__class__.__name__, self.identifier) - - def keys(self): - return self.state.keys() - - def copy(self): - copyDict = {} - for item in self.keys(): - copyDict[item] = self[item] - return copyDict - - def get(self, item, default): - return self.state.get(item, default) - - def update(self, update_dict): - return self.state.update(update_dict) - - def setdefault(self, item, default): - return self.state.setdefault(item, default) - - def __contains__(self, item): - return item in self.state - - def __getitem__(self, item): - return self.state[item] - - def __setitem__(self, item, val): - self.state[item] = val - return self.state[item] - - def __delitem__(self, item): - """We don't support this""" - raise Exception("Silly monkey, models NEED all their properties.") - - def is_new_record(self): - return self.initial_state == {} - - @absorb_connection_error - def add_to_index(self): - """Each insance of Foo has its id tracked int the set named Foos""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().sadd(set_name, self.identifier) - - @absorb_connection_error - def remove_from_index(self): - """Remove id of this instance from the set tracking ids of this type""" - set_name = self.__class__._redis_set_name(self.__class__.__name__) - Redis.instance().srem(set_name, self.identifier) - - @absorb_connection_error - def associate_with(self, foreign_type, foreign_id): - """Add this class id into the set foreign_type:foreign_id:this_types""" - # note the extra 's' on the end is for plurality - # to match the old data without requiring a migration of any sort - self.add_associated_model_to_its_set(foreign_type, foreign_id) - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().sadd(redis_set, self.identifier) - - @absorb_connection_error - def unassociate_with(self, foreign_type, foreign_id): - """Delete from foreign_type:foreign_id:this_types set""" - redis_set = self.__class__._redis_association_name(foreign_type, - foreign_id) - Redis.instance().srem(redis_set, self.identifier) - - def add_associated_model_to_its_set(self, model_type, model_id): - """ - When associating an X to a Y, save Y for newer timestamp, etc, and to - make sure to save it if Y is a new record. - If the model_type isn't found as a usable class, ignore it, this can - happen when associating to things stored in LDAP (user, project, ...). - """ - table = globals() - klsname = model_type.capitalize() - if table.has_key(klsname): - model_class = table[klsname] - model_inst = model_class(model_id) - model_inst.save() - - @absorb_connection_error - def save(self): - """ - update the directory with the state from this model - also add it to the index of items of the same type - then set the initial_state = state so new changes are tracked - """ - # TODO(ja): implement hmset in redis-py and use it - # instead of multiple calls to hset - if self.is_new_record(): - self["create_time"] = utils.isotime() - for key, val in self.state.iteritems(): - Redis.instance().hset(self.__redis_key, key, val) - self.add_to_index() - self.initial_state = dict(self.state) - return True - - @absorb_connection_error - def destroy(self): - """deletes all related records from datastore.""" - logging.info("Destroying datamodel for %s %s", - self.__class__.__name__, self.identifier) - Redis.instance().delete(self.__redis_key) - self.remove_from_index() - return True - diff --git a/nova/tests/model_unittest.py b/nova/tests/model_unittest.py deleted file mode 100644 index 130516c6..00000000 --- a/nova/tests/model_unittest.py +++ /dev/null @@ -1,292 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from datetime import datetime, timedelta -import logging -import time - -from nova import flags -from nova import test -from nova import utils -from nova.compute import model - - -FLAGS = flags.FLAGS - - -class ModelTestCase(test.TrialTestCase): - def setUp(self): - super(ModelTestCase, self).setUp() - self.flags(connection_type='fake', - fake_storage=True) - - def tearDown(self): - model.Instance('i-test').destroy() - model.Host('testhost').destroy() - model.Daemon('testhost', 'nova-testdaemon').destroy() - - def create_instance(self): - inst = model.Instance('i-test') - inst['reservation_id'] = 'r-test' - inst['launch_time'] = '10' - inst['user_id'] = 'fake' - inst['project_id'] = 'fake' - inst['instance_type'] = 'm1.tiny' - inst['mac_address'] = utils.generate_mac() - inst['ami_launch_index'] = 0 - inst['private_dns_name'] = '10.0.0.1' - inst.save() - return inst - - def create_host(self): - host = model.Host('testhost') - host.save() - return host - - def create_daemon(self): - daemon = model.Daemon('testhost', 'nova-testdaemon') - daemon.save() - return daemon - - def create_session_token(self): - session_token = model.SessionToken('tk12341234') - session_token['user'] = 'testuser' - session_token.save() - return session_token - - def test_create_instance(self): - """store with create_instace, then test that a load finds it""" - instance = self.create_instance() - old = model.Instance(instance.identifier) - self.assertFalse(old.is_new_record()) - - def test_delete_instance(self): - """create, then destroy, then make sure loads a new record""" - instance = self.create_instance() - instance.destroy() - newinst = model.Instance('i-test') - self.assertTrue(newinst.is_new_record()) - - def test_instance_added_to_set(self): - """create, then check that it is listed in global set""" - instance = self.create_instance() - found = False - for x in model.InstanceDirectory().all: - if x.identifier == 'i-test': - found = True - self.assert_(found) - - def test_instance_associates_project(self): - """create, then check that it is listed for the project""" - instance = self.create_instance() - found = False - for x in model.InstanceDirectory().by_project(instance.project): - if x.identifier == 'i-test': - found = True - self.assert_(found) - - def test_instance_associates_ip(self): - """create, then check that it is listed for the ip""" - instance = self.create_instance() - found = False - x = model.InstanceDirectory().by_ip(instance['private_dns_name']) - self.assertEqual(x.identifier, 'i-test') - - def test_instance_associates_node(self): - """create, then check that it is listed for the host""" - instance = self.create_instance() - found = False - for x in model.InstanceDirectory().by_node(FLAGS.host): - if x.identifier == 'i-test': - found = True - self.assertFalse(found) - instance['host'] = 'test_node' - instance.save() - for x in model.InstanceDirectory().by_node('test_node'): - if x.identifier == 'i-test': - found = True - self.assert_(found) - - - def test_host_class_finds_hosts(self): - host = self.create_host() - self.assertEqual('testhost', model.Host.lookup('testhost').identifier) - - def test_host_class_doesnt_find_missing_hosts(self): - rv = model.Host.lookup('woahnelly') - self.assertEqual(None, rv) - - def test_create_host(self): - """store with create_host, then test that a load finds it""" - host = self.create_host() - old = model.Host(host.identifier) - self.assertFalse(old.is_new_record()) - - def test_delete_host(self): - """create, then destroy, then make sure loads a new record""" - instance = self.create_host() - instance.destroy() - newinst = model.Host('testhost') - self.assertTrue(newinst.is_new_record()) - - def test_host_added_to_set(self): - """create, then check that it is included in list""" - instance = self.create_host() - found = False - for x in model.Host.all(): - if x.identifier == 'testhost': - found = True - self.assert_(found) - - def test_create_daemon_two_args(self): - """create a daemon with two arguments""" - d = self.create_daemon() - d = model.Daemon('testhost', 'nova-testdaemon') - self.assertFalse(d.is_new_record()) - - def test_create_daemon_single_arg(self): - """Create a daemon using the combined host:bin format""" - d = model.Daemon("testhost:nova-testdaemon") - d.save() - d = model.Daemon('testhost:nova-testdaemon') - self.assertFalse(d.is_new_record()) - - def test_equality_of_daemon_single_and_double_args(self): - """Create a daemon using the combined host:bin arg, find with 2""" - d = model.Daemon("testhost:nova-testdaemon") - d.save() - d = model.Daemon('testhost', 'nova-testdaemon') - self.assertFalse(d.is_new_record()) - - def test_equality_daemon_of_double_and_single_args(self): - """Create a daemon using the combined host:bin arg, find with 2""" - d = self.create_daemon() - d = model.Daemon('testhost:nova-testdaemon') - self.assertFalse(d.is_new_record()) - - def test_delete_daemon(self): - """create, then destroy, then make sure loads a new record""" - instance = self.create_daemon() - instance.destroy() - newinst = model.Daemon('testhost', 'nova-testdaemon') - self.assertTrue(newinst.is_new_record()) - - def test_daemon_heartbeat(self): - """Create a daemon, sleep, heartbeat, check for update""" - d = self.create_daemon() - ts = d['updated_at'] - time.sleep(2) - d.heartbeat() - d2 = model.Daemon('testhost', 'nova-testdaemon') - ts2 = d2['updated_at'] - self.assert_(ts2 > ts) - - def test_daemon_added_to_set(self): - """create, then check that it is included in list""" - instance = self.create_daemon() - found = False - for x in model.Daemon.all(): - if x.identifier == 'testhost:nova-testdaemon': - found = True - self.assert_(found) - - def test_daemon_associates_host(self): - """create, then check that it is listed for the host""" - instance = self.create_daemon() - found = False - for x in model.Daemon.by_host('testhost'): - if x.identifier == 'testhost:nova-testdaemon': - found = True - self.assertTrue(found) - - def test_create_session_token(self): - """create""" - d = self.create_session_token() - d = model.SessionToken(d.token) - self.assertFalse(d.is_new_record()) - - def test_delete_session_token(self): - """create, then destroy, then make sure loads a new record""" - instance = self.create_session_token() - instance.destroy() - newinst = model.SessionToken(instance.token) - self.assertTrue(newinst.is_new_record()) - - def test_session_token_added_to_set(self): - """create, then check that it is included in list""" - instance = self.create_session_token() - found = False - for x in model.SessionToken.all(): - if x.identifier == instance.token: - found = True - self.assert_(found) - - def test_session_token_associates_user(self): - """create, then check that it is listed for the user""" - instance = self.create_session_token() - found = False - for x in model.SessionToken.associated_to('user', 'testuser'): - if x.identifier == instance.identifier: - found = True - self.assertTrue(found) - - def test_session_token_generation(self): - instance = model.SessionToken.generate('username', 'TokenType') - self.assertFalse(instance.is_new_record()) - - def test_find_generated_session_token(self): - instance = model.SessionToken.generate('username', 'TokenType') - found = model.SessionToken.lookup(instance.identifier) - self.assert_(found) - - def test_update_session_token_expiry(self): - instance = model.SessionToken('tk12341234') - oldtime = datetime.utcnow() - instance['expiry'] = oldtime.strftime(utils.TIME_FORMAT) - instance.update_expiry() - expiry = utils.parse_isotime(instance['expiry']) - self.assert_(expiry > datetime.utcnow()) - - def test_session_token_lookup_when_expired(self): - instance = model.SessionToken.generate("testuser") - instance['expiry'] = datetime.utcnow().strftime(utils.TIME_FORMAT) - instance.save() - inst = model.SessionToken.lookup(instance.identifier) - self.assertFalse(inst) - - def test_session_token_lookup_when_not_expired(self): - instance = model.SessionToken.generate("testuser") - inst = model.SessionToken.lookup(instance.identifier) - self.assert_(inst) - - def test_session_token_is_expired_when_expired(self): - instance = model.SessionToken.generate("testuser") - instance['expiry'] = datetime.utcnow().strftime(utils.TIME_FORMAT) - self.assert_(instance.is_expired()) - - def test_session_token_is_expired_when_not_expired(self): - instance = model.SessionToken.generate("testuser") - self.assertFalse(instance.is_expired()) - - def test_session_token_ttl(self): - instance = model.SessionToken.generate("testuser") - now = datetime.utcnow() - delta = timedelta(hours=1) - instance['expiry'] = (now + delta).strftime(utils.TIME_FORMAT) - # give 5 seconds of fuzziness - self.assert_(abs(instance.ttl() - FLAGS.auth_token_ttl) < 5) diff --git a/run_tests.py b/run_tests.py index c47cbe2e..d5dc5f93 100644 --- a/run_tests.py +++ b/run_tests.py @@ -55,7 +55,6 @@ from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * -#from nova.tests.model_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * From e8709043a214907cae7dc56c57dc1566fce8f527 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 14:57:45 -0700 Subject: [PATCH 073/113] fix service unit tests --- nova/tests/service_unittest.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 274e74b5..590d760b 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -47,34 +47,50 @@ class ServiceTestCase(test.BaseTestCase): self.mox.StubOutWithMock(service, 'db') def test_create(self): + host='foo' + binary='nova-fake' + topic='fake' self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) self.mox.StubOutWithMock( service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='fake', + topic=topic, proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='fake.%s' % FLAGS.host, + topic='%s.%s' % (topic, host), proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) # Stub out looping call a bit needlessly since we don't have an easy # way to cancel it (yet) when the tests finishes - service.task.LoopingCall( - mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn( + service.task.LoopingCall(mox.IgnoreArg()).AndReturn( service.task.LoopingCall) service.task.LoopingCall.start(interval=mox.IgnoreArg(), now=mox.IgnoreArg()) rpc.AdapterConsumer.attach_to_twisted() rpc.AdapterConsumer.attach_to_twisted() + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} + service_ref = {'host': host, + 'binary': binary, + 'report_count': 0, + 'id': 1} + + service.db.service_get_by_args(None, + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(None, + service_create).AndReturn(service_ref['id']) self.mox.ReplayAll() - app = service.Service.create(bin_name='nova-fake') + app = service.Service.create(host=host, binary=binary) self.assert_(app) # We're testing sort of weird behavior in how report_state decides From e8fa82f74efa22d6923a767af2d5ace5b74ecd71 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 23:04:41 -0700 Subject: [PATCH 074/113] removed references to compute.model --- nova/endpoint/admin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py index d6f62275..3d91c66d 100644 --- a/nova/endpoint/admin.py +++ b/nova/endpoint/admin.py @@ -22,8 +22,9 @@ Admin API controller, exposed through http via the api worker. import base64 +from nova import db +from nova import exception from nova.auth import manager -from nova.compute import model def user_dict(user, base64_file=None): @@ -52,6 +53,7 @@ def project_dict(project): def host_dict(host): """Convert a host model object to a result dict""" if host: + # FIXME(vish) return host.state else: return {} @@ -181,7 +183,7 @@ class AdminController(object): result = { 'members': [{'member': m} for m in project.member_ids]} return result - + @admin_only def modify_project_member(self, context, user, project, operation, **kwargs): """Add or remove a user from a project.""" @@ -203,9 +205,9 @@ class AdminController(object): * DHCP servers running * Iptables / bridges """ - return {'hostSet': [host_dict(h) for h in model.Host.all()]} + return {'hostSet': [host_dict(h) for h in db.host_get_all()]} @admin_only def describe_host(self, _context, name, **_kwargs): """Returns status info for single node.""" - return host_dict(model.Host.lookup(name)) + return host_dict(db.host_get(name)) From fa09ff955efdc5e1022848b4b82325f32aa282b1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 23:13:12 -0700 Subject: [PATCH 075/113] removed model from nova-manage --- bin/nova-manage | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 7f20531d..055f2c3a 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -25,10 +25,10 @@ import sys import time +from nova import db from nova import flags from nova import utils from nova.auth import manager -from nova.compute import model from nova.cloudpipe import pipelib from nova.endpoint import cloud @@ -41,7 +41,6 @@ class VpnCommands(object): def __init__(self): self.manager = manager.AuthManager() - self.instdir = model.InstanceDirectory() self.pipe = pipelib.CloudPipe(cloud.CloudController()) def list(self): @@ -73,9 +72,8 @@ class VpnCommands(object): def _vpn_for(self, project_id): """Get the VPN instance for a project ID.""" - for instance in self.instdir.all: - if ('image_id' in instance.state - and instance['image_id'] == FLAGS.vpn_image_id + for instance in db.instance_get_all(): + if (instance['image_id'] == FLAGS.vpn_image_id and not instance['state_description'] in ['shutting_down', 'shutdown'] and instance['project_id'] == project_id): From 66d834a68ff02b983e1803b8e661bfa58a0bc175 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 3 Sep 2010 00:11:59 -0700 Subject: [PATCH 076/113] reverting accidental search/replace change to server.py --- nova/server.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nova/server.py b/nova/server.py index 8cc1e0ff..d4563bfe 100644 --- a/nova/server.py +++ b/nova/server.py @@ -17,11 +17,11 @@ # under the License. """ -Base functionality for nova services - gradually being replaced with twistd.py. +Base functionality for nova daemons - gradually being replaced with twistd.py. """ -import service -from service import pidlockfile +import daemon +from daemon import pidlockfile import logging import logging.handlers import os @@ -33,14 +33,14 @@ from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_bool('serviceize', False, 'serviceize this process') -# NOTE(termie): right now I am defaulting to using syslog when we serviceize +flags.DEFINE_bool('daemonize', False, 'daemonize this process') +# NOTE(termie): right now I am defaulting to using syslog when we daemonize # it may be better to do something else -shrug- # NOTE(Devin): I think we should let each process have its own log file # and put it in /var/logs/nova/(appname).log # This makes debugging much easier and cuts down on sys log # clutter. -flags.DEFINE_bool('use_syslog', True, 'output to syslog when serviceizing') +flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') flags.DEFINE_string('logfile', None, 'log file to output to') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') @@ -50,17 +50,17 @@ flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') def stop(pidfile): """ - Stop the service + Stop the daemon """ # Get the pid from the pidfile try: pid = int(open(pidfile,'r').read().strip()) except IOError: - message = "pidfile %s does not exist. Service not running?\n" + message = "pidfile %s does not exist. Daemon not running?\n" sys.stderr.write(message % pidfile) return # not an error in a restart - # Try killing the service process + # Try killing the daemon process try: while 1: os.kill(pid, signal.SIGTERM) @@ -100,13 +100,13 @@ def serve(name, main): else: print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - serviceize(argv, name, main) + daemonize(argv, name, main) -def serviceize(args, name, main): - """Does the work of serviceizing the process""" +def daemonize(args, name, main): + """Does the work of daemonizing the process""" logging.getLogger('amqplib').setLevel(logging.WARN) - if FLAGS.serviceize: + if FLAGS.daemonize: logger = logging.getLogger() formatter = logging.Formatter( name + '(%(name)s): %(levelname)s %(message)s') @@ -129,8 +129,8 @@ def serviceize(args, name, main): else: logging.getLogger().setLevel(logging.WARNING) - with service.ServiceContext( - detach_process=FLAGS.serviceize, + with daemon.DaemonContext( + detach_process=FLAGS.daemonize, working_directory=FLAGS.working_directory, pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, acquire_timeout=1, From b627a37a956b91e1210188f50d718c1a9845672b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 3 Sep 2010 00:28:16 -0700 Subject: [PATCH 077/113] fixed up format_instances --- nova/endpoint/cloud.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index c34eb5da..15136ada 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -380,30 +380,30 @@ class CloudController(object): } floating_addr = db.instance_get_floating_address(context, instance['id']) - i['public_dns_name'] = floating_addr + i['publicDnsName'] = floating_addr fixed_addr = db.instance_get_fixed_address(context, instance['id']) - i['private_dns_name'] = fixed_addr - if not i['public_dns_name']: - i['public_dns_name'] = i['private_dns_name'] - i['dns_name'] = None - i['key_name'] = instance.key_name + i['privateDnsName'] = fixed_addr + if not i['publicDnsName']: + i['publicDnsName'] = i['privateDnsName'] + i['dnsName'] = None + i['keyName'] = instance['key_name'] if context.user.is_admin(): - i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.project_id, - 'host') # FIXME - i['product_codes_set'] = self._convert_to_set([], 'product_codes') - i['instance_type'] = instance.instance_type - i['launch_time'] = instance.created_at - i['ami_launch_index'] = instance.launch_index + i['keyName'] = '%s (%s, %s)' % (i['keyName'], + instance['project_id'], + instance['host']) + i['productCodesSet'] = self._convert_to_set([], 'product_codes') + i['instanceType'] = instance['instance_type'] + i['launchTime'] = instance['created_at'] + i['amiLaunchIndex'] = instance['launch_index'] if not reservations.has_key(instance['reservation_id']): r = {} - r['reservation_id'] = instance['reservation_id'] - r['owner_id'] = instance.project_id - r['group_set'] = self._convert_to_set([], 'groups') - r['instances_set'] = [] + r['reservationId'] = instance['reservation_id'] + r['ownerId'] = instance['project_id'] + r['groupSet'] = self._convert_to_set([], 'groups') + r['instancesSet'] = [] reservations[instance['reservation_id']] = r - reservations[instance['reservation_id']]['instances_set'].append(i) + reservations[instance['reservation_id']]['instancesSet'].append(i) return list(reservations.values()) From 676769f786dac746dbe1d187cf03b701f918850b Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 5 Sep 2010 05:56:53 +0100 Subject: [PATCH 078/113] Bug #630640: Duplicated power state constants Remove power state constants that have ended up duplicated following a bad merge. They were moved from nova.compute.node.Instance into nova.compute.power_state at the same time that Instance was moved into nova.compute.service. We've ended up with these constants in both places. Remove the ones from service, in favour of the ones in power_state. --- nova/tests/cloud_unittest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 900ff5a9..19aa23b9 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -28,6 +28,7 @@ from nova import flags from nova import rpc from nova import test from nova.auth import manager +from nova.compute import power_state from nova.compute import service from nova.endpoint import api from nova.endpoint import cloud @@ -95,7 +96,7 @@ class CloudTestCase(test.BaseTestCase): rv = yield defer.succeed(time.sleep(1)) info = self.cloud._get_instance(instance['instance_id']) logging.debug(info['state']) - if info['state'] == node.Instance.RUNNING: + if info['state'] == power_state.RUNNING: break self.assert_(rv) From 9025dc8f9918d59a2e9d9908b3000f331488c708 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 05:26:08 -0700 Subject: [PATCH 079/113] Lots of fixes to make the nova commands work properly and make datamodel work with mysql properly --- nova/endpoint/cloud.py | 11 ++--- nova/process.py | 95 +++++++++++++++++++++--------------------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 15136ada..932d42de 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -41,6 +41,7 @@ from nova.endpoint import images FLAGS = flags.FLAGS +flags.DECLARE('storage_availability_zone', 'nova.volume.manager') def _gen_key(user_id, key_name): @@ -262,11 +263,11 @@ class CloudController(object): volume['mountpoint']) if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], - 'deleteOnTermination': volume['delete_on_termination'], + 'deleteOnTermination': False, 'device': volume['mountpoint'], 'instanceId': volume['instance_id'], 'status': 'attached', - 'volume_id': volume['volume_id']}] + 'volume_id': volume['str_id']}] else: v['attachmentSet'] = [{}] return v @@ -293,7 +294,7 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_str(context, volume_id) # TODO(vish): abstract status checking? - if volume_ref['status'] == "attached": + if volume_ref['attach_status'] == "attached": raise exception.ApiError("Volume is already attached") #volume.start_attach(instance_id, device) instance_ref = db.instance_get_by_str(context, instance_id) @@ -306,7 +307,7 @@ class CloudController(object): "mountpoint": device}}) return defer.succeed({'attachTime': volume_ref['attach_time'], 'device': volume_ref['mountpoint'], - 'instanceId': instance_ref['id_str'], + 'instanceId': instance_ref['id'], 'requestId': context.request_id, 'status': volume_ref['attach_status'], 'volumeId': volume_ref['id']}) @@ -334,7 +335,7 @@ class CloudController(object): db.volume_detached(context) return defer.succeed({'attachTime': volume_ref['attach_time'], 'device': volume_ref['mountpoint'], - 'instanceId': instance_ref['id_str'], + 'instanceId': instance_ref['str_id'], 'requestId': context.request_id, 'status': volume_ref['attach_status'], 'volumeId': volume_ref['id']}) diff --git a/nova/process.py b/nova/process.py index 425d9f16..74725c15 100644 --- a/nova/process.py +++ b/nova/process.py @@ -18,9 +18,10 @@ # under the License. """ -Process pool, still buggy right now. +Process pool using twisted threading """ +import logging import StringIO from twisted.internet import defer @@ -29,30 +30,14 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags +from nova.utils import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') - -# NOTE(termie): this is copied from twisted.internet.utils but since -# they don't export it I've copied and modified -class UnexpectedErrorOutput(IOError): - """ - Standard error data was received where it was not expected. This is a - subclass of L{IOError} to preserve backward compatibility with the previous - error behavior of L{getProcessOutput}. - - @ivar processEnded: A L{Deferred} which will fire when the process which - produced the data on stderr has ended (exited and all file descriptors - closed). - """ - def __init__(self, stdout=None, stderr=None): - IOError.__init__(self, "got stdout: %r\nstderr: %r" % (stdout, stderr)) - - -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to +# This is based on _BackRelay from twister.internal.utils, but modified to +# capture both stdout and stderr, without odd stderr handling, and also to # handle stdin class BackRelayWithInput(protocol.ProcessProtocol): """ @@ -62,22 +47,23 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_UnexpectedErrorOutput} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_UnexpectedErrorOutput} which C{deferred} fires - with earlier in this case so that users can determine when the process + @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are + received over stderr, this attribute will refer to a L{Deferred} which + will be called back when the process ends. This C{Deferred} is also + associated with the L{_ProcessExecutionError} which C{deferred} fires + with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): + def __init__(self, deferred, cmd, started_deferred=None, + terminate_on_stderr=False, check_exit_code=True, + process_input=None): self.deferred = deferred + self.cmd = cmd self.stdout = StringIO.StringIO() self.stderr = StringIO.StringIO() self.started_deferred = started_deferred @@ -85,14 +71,18 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + + def _build_execution_error(self, exit_code=None): + return ProcessExecutionError(cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) + def errReceived(self, text): self.stderr.write(text) if self.terminate_on_stderr and (self.deferred is not None): self.on_process_ended = defer.Deferred() - self.deferred.errback(UnexpectedErrorOutput( - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue())) + self.deferred.errback(self._build_execution_error()) self.deferred = None self.transport.loseConnection() @@ -102,15 +92,19 @@ class BackRelayWithInput(protocol.ProcessProtocol): def processEnded(self, reason): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(UnexpectedErrorOutput(stdout, stderr)) + exit_code = reason.value.exitCode + if self.check_exit_code and exit_code <> 0: + self.deferred.errback(self._build_execution_error(exit_code)) + else: + try: + if self.check_exit_code: + reason.trap(error.ProcessDone) + self.deferred.callback((stdout, stderr)) + except: + # NOTE(justinsb): This logic is a little suspicious to me... + # If the callback throws an exception, then errback will be + # called also. However, this is what the unit tests test for... + self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) @@ -122,8 +116,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(self.process_input) self.transport.closeStdin() -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, +def get_process_output(executable, args=None, env=None, path=None, + process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, terminate_on_stderr=False): if process_reactor is None: @@ -131,10 +125,15 @@ def get_process_output(executable, args=None, env=None, path=None, args = args and args or () env = env and env and {} deferred = defer.Deferred() + cmd = executable + if args: + cmd = cmd + " " + ' '.join(args) + logging.debug("Running cmd: %s", cmd) process_handler = BackRelayWithInput( - deferred, - started_deferred=started_deferred, - check_exit_code=check_exit_code, + deferred, + cmd, + started_deferred=started_deferred, + check_exit_code=check_exit_code, process_input=process_input, terminate_on_stderr=terminate_on_stderr) # NOTE(vish): commands come in as unicode, but self.executes needs @@ -142,7 +141,7 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, + process_reactor.spawnProcess( process_handler, executable, (executable,)+tuple(args), env, path) return deferred From 2aba4b149c94f200811ef3a40ba77d2f2008e04c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 12:37:18 -0700 Subject: [PATCH 080/113] fix floating_ip to follow standard create pattern --- nova/tests/network_unittest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index f3124c1b..8e462b9d 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -87,11 +87,12 @@ class NetworkTestCase(test.TrialTestCase): """Makes sure that we can allocaate a public ip""" # TODO(vish): better way of adding floating ips pubnet = IPy.IP(flags.FLAGS.public_range) - ip_str = str(pubnet[0]) + address = str(pubnet[0]) try: - db.floating_ip_get_by_address(None, ip_str) + db.floating_ip_get_by_address(None, address) except exception.NotFound: - db.floating_ip_create(None, ip_str, FLAGS.host) + db.floating_ip_create(None, {'address': address, + 'host': FLAGS.host}) float_addr = self.network.allocate_floating_ip(self.context, self.projects[0].id) fix_addr = self._create_address(0) From 993e5121a5008e46b9f6d36007d8dac20fa95826 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 19:48:38 -0700 Subject: [PATCH 081/113] dhcpbridge fixes from review --- bin/nova-dhcpbridge | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index c416d07a..980da1ce 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -34,12 +34,13 @@ from nova import db from nova import flags from nova import rpc from nova import utils -from nova import datastore # for redis_db flag -from nova.auth import manager # for auth flags from nova.network import linux_net -from nova.network import manager # for network flags FLAGS = flags.FLAGS +flags.DECLARE('auth_driver', 'nova.auth.manager') +flags.DECLARE('redis_db', 'nova.datastore') +flags.DECLARE('network_size', 'nova.network.manager') +flags.DECLARE('num_networks', 'nova.network.manager') def add_lease(_mac, ip_address, _hostname, _interface): @@ -80,7 +81,6 @@ def init_leases(interface): def main(): - global network_manager """Parse environment and arguments and call the approproate action.""" flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) @@ -99,7 +99,6 @@ def main(): '_trial_temp', 'nova.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path - #FLAGS.sql_connection = 'mysql://root@localhost/test' action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] From 7f396cbd975d9a4302cd626e430b6eb5fff30943 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 20:55:42 -0700 Subject: [PATCH 082/113] more fixes from code review --- nova/endpoint/admin.py | 3 +- nova/endpoint/cloud.py | 72 ++++++++++++++++++++---------------------- nova/flags.py | 11 +++---- 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py index 3d91c66d..c6dcb532 100644 --- a/nova/endpoint/admin.py +++ b/nova/endpoint/admin.py @@ -53,7 +53,6 @@ def project_dict(project): def host_dict(host): """Convert a host model object to a result dict""" if host: - # FIXME(vish) return host.state else: return {} @@ -195,6 +194,8 @@ class AdminController(object): raise exception.ApiError('operation must be add or remove') return True + # FIXME(vish): these host commands don't work yet, perhaps some of the + # required data can be retrieved from service objects? @admin_only def describe_hosts(self, _context, **_kwargs): """Returns status info for all nodes. Includes: diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 932d42de..709c967b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -78,7 +78,7 @@ class CloudController(object): if not os.path.exists(root_ca_path): start = os.getcwd() os.chdir(FLAGS.ca_path) - # TODO: Do this with M2Crypto instead + # TODO(vish): Do this with M2Crypto instead utils.runthis("Generating root CA: %s", "sh genrootca.sh") os.chdir(start) @@ -93,28 +93,30 @@ class CloudController(object): result[instance['key_name']] = [line] return result - def get_metadata(self, ipaddress): - i = db.fixed_ip_get_instance(ipaddress) - if i is None: + def get_metadata(self, address): + instance_ref = db.fixed_ip_get_instance(None, address) + if instance_ref is None: return None - mpi = self._get_mpi_data(i['project_id']) - if i['key_name']: + mpi = self._get_mpi_data(instance_ref['project_id']) + if instance_ref['key_name']: keys = { '0': { - '_name': i['key_name'], - 'openssh-key': i['key_data'] + '_name': instance_ref['key_name'], + 'openssh-key': instance_ref['key_data'] } } else: keys = '' - hostname = i['hostname'] + hostname = instance_ref['hostname'] + floating_ip = db.instance_get_floating_ip_address(None, + instance_ref['id']) data = { - 'user-data': base64.b64decode(i['user_data']), + 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { - 'ami-id': i['image_id'], - 'ami-launch-index': i['ami_launch_index'], - 'ami-manifest-path': 'FIXME', # image property - 'block-device-mapping': { # TODO: replace with real data + 'ami-id': instance_ref['image_id'], + 'ami-launch-index': instance_ref['ami_launch_index'], + 'ami-manifest-path': 'FIXME', + 'block-device-mapping': { # TODO(vish): replace with real data 'ami': 'sda1', 'ephemeral0': 'sda2', 'root': '/dev/sda1', @@ -122,27 +124,27 @@ class CloudController(object): }, 'hostname': hostname, 'instance-action': 'none', - 'instance-id': i['instance_id'], - 'instance-type': i.get('instance_type', ''), + 'instance-id': instance_ref['str_id'], + 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, - 'local-ipv4': i['private_dns_name'], # TODO: switch to IP - 'kernel-id': i.get('kernel_id', ''), + 'local-ipv4': address, + 'kernel-id': instance_ref['kernel_id'], 'placement': { - 'availaibility-zone': i.get('availability_zone', 'nova'), + 'availaibility-zone': instance_ref['availability_zone'], }, 'public-hostname': hostname, - 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP + 'public-ipv4': floating_ip or '', 'public-keys': keys, - 'ramdisk-id': i.get('ramdisk_id', ''), - 'reservation-id': i['reservation_id'], - 'security-groups': i.get('groups', ''), + 'ramdisk-id': instance_ref['ramdisk_id'], + 'reservation-id': instance_ref['reservation_id'], + 'security-groups': '', 'mpi': mpi } } - if False: # TODO: store ancestor ids + if False: # TODO(vish): store ancestor ids data['ancestor-ami-ids'] = [] - if i.get('product_codes', None): - data['product-codes'] = i['product_codes'] + if False: # TODO(vish): store product codes + data['product-codes'] = [] return data @rbac.allow('all') @@ -253,7 +255,7 @@ class CloudController(object): v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] - # v['createTime'] = volume['create_time'] + v['createTime'] = volume['created_at'] if context.user.is_admin(): v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], @@ -296,7 +298,6 @@ class CloudController(object): # TODO(vish): abstract status checking? if volume_ref['attach_status'] == "attached": raise exception.ApiError("Volume is already attached") - #volume.start_attach(instance_id, device) instance_ref = db.instance_get_by_str(context, instance_id) host = db.instance_get_host(context, instance_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), @@ -322,7 +323,6 @@ class CloudController(object): if volume_ref['status'] == "available": raise exception.Error("Volume is already detached") try: - #volume.start_detach() host = db.instance_get_host(context, instance_ref['id']) rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "detach_volume", @@ -422,15 +422,12 @@ class CloudController(object): for floating_ip_ref in iterator: address = floating_ip_ref['id_str'] instance_ref = db.floating_ip_get_instance(address) - address_rv = { - 'public_ip': address, - 'instance_id': instance_ref['id_str'] - } + address_rv = {'public_ip': address, + 'instance_id': instance_ref['id_str']} if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s)" % ( - address_rv['instance_id'], - floating_ip_ref['project_id'], - ) + details = "%s (%s)" % (address_rv['instance_id'], + floating_ip_ref['project_id']) + address_rv['instance_id'] = details addresses.append(address_rv) return {'addressesSet': addresses} @@ -579,7 +576,6 @@ class CloudController(object): @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") - # network_topic = yield self._get_network_topic(context) for id_str in instance_id: logging.debug("Going to try and terminate %s" % id_str) try: diff --git a/nova/flags.py b/nova/flags.py index ebbfe3ff..7b0c95a3 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -203,12 +203,6 @@ DEFINE_string('vpn_key_suffix', DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') -# UNUSED -DEFINE_string('node_availability_zone', 'nova', - 'availability zone of this node') -DEFINE_string('host', socket.gethostname(), - 'name of this node') - DEFINE_string('sql_connection', 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), 'connection string for sql database') @@ -220,4 +214,9 @@ DEFINE_string('network_manager', 'nova.network.manager.VlanManager', DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager', 'Manager for volume') +DEFINE_string('host', socket.gethostname(), + 'name of this node') +# UNUSED +DEFINE_string('node_availability_zone', 'nova', + 'availability zone of this node') From ced9e5c62f5c05d9cec03d14e77b0f2df7af3a95 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 21:15:22 -0700 Subject: [PATCH 083/113] Last of cleanup, including removing fake_storage flage --- nova/tests/access_unittest.py | 2 - nova/tests/auth_unittest.py | 3 +- nova/tests/cloud_unittest.py | 3 +- nova/tests/compute_unittest.py | 3 +- nova/tests/fake_flags.py | 5 +- nova/tests/network_unittest.py | 1 - nova/tests/real_flags.py | 1 - nova/tests/service_unittest.py | 72 ++++++++++----------- nova/tests/storage_unittest.py | 115 --------------------------------- nova/tests/volume_unittest.py | 3 +- 10 files changed, 42 insertions(+), 166 deletions(-) delete mode 100644 nova/tests/storage_unittest.py diff --git a/nova/tests/access_unittest.py b/nova/tests/access_unittest.py index fa0a090a..59e1683d 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/access_unittest.py @@ -33,8 +33,6 @@ class Context(object): class AccessTestCase(test.BaseTestCase): def setUp(self): super(AccessTestCase, self).setUp() - FLAGS.connection_type = 'fake' - FLAGS.fake_storage = True um = manager.AuthManager() # Make test users try: diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 59a81818..b54e6827 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -34,8 +34,7 @@ FLAGS = flags.FLAGS class AuthTestCase(test.BaseTestCase): def setUp(self): super(AuthTestCase, self).setUp() - self.flags(connection_type='fake', - fake_storage=True) + self.flags(connection_type='fake') self.manager = manager.AuthManager() def test_001_can_create_users(self): diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index e6796e3d..29947e03 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -39,8 +39,7 @@ FLAGS = flags.FLAGS class CloudTestCase(test.BaseTestCase): def setUp(self): super(CloudTestCase, self).setUp() - self.flags(connection_type='fake', - fake_storage=True) + self.flags(connection_type='fake') self.conn = rpc.Connection.instance() logging.getLogger().setLevel(logging.DEBUG) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 867b572f..07a2fceb 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -38,8 +38,7 @@ class ComputeTestCase(test.TrialTestCase): def setUp(self): # pylint: disable-msg=C0103 logging.getLogger().setLevel(logging.DEBUG) super(ComputeTestCase, self).setUp() - self.flags(connection_type='fake', - fake_storage=True) + self.flags(connection_type='fake') self.compute = utils.import_object(FLAGS.compute_manager) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 3114912b..8f475465 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -20,8 +20,8 @@ from nova import flags FLAGS = flags.FLAGS -flags.DECLARE('fake_storage', 'nova.volume.manager') -FLAGS.fake_storage = True +flags.DECLARE('volume_driver', 'nova.volume.manager') +FLAGS.volume_driver = 'nova.volume.driver.FakeAOEDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' @@ -37,4 +37,3 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' -#FLAGS.sql_connection = 'mysql://root@localhost/test' diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 8e462b9d..a89f1d62 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -39,7 +39,6 @@ class NetworkTestCase(test.TrialTestCase): # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge self.flags(connection_type='fake', - fake_storage=True, fake_network=True, auth_driver='nova.auth.ldapdriver.FakeLdapDriver', network_size=16, diff --git a/nova/tests/real_flags.py b/nova/tests/real_flags.py index 121f4eb4..71da0499 100644 --- a/nova/tests/real_flags.py +++ b/nova/tests/real_flags.py @@ -21,7 +21,6 @@ from nova import flags FLAGS = flags.FLAGS FLAGS.connection_type = 'libvirt' -FLAGS.fake_storage = False FLAGS.fake_rabbit = False FLAGS.fake_network = False FLAGS.verbose = False diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 590d760b..097a045e 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -47,9 +47,9 @@ class ServiceTestCase(test.BaseTestCase): self.mox.StubOutWithMock(service, 'db') def test_create(self): - host='foo' - binary='nova-fake' - topic='fake' + host = 'foo' + binary = 'nova-fake' + topic = 'fake' self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) @@ -75,19 +75,19 @@ class ServiceTestCase(test.BaseTestCase): rpc.AdapterConsumer.attach_to_twisted() rpc.AdapterConsumer.attach_to_twisted() service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0} + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'report_count': 0, + 'id': 1} service.db.service_get_by_args(None, - host, - binary).AndRaise(exception.NotFound()) + host, + binary).AndRaise(exception.NotFound()) service.db.service_create(None, - service_create).AndReturn(service_ref['id']) + service_create).AndReturn(service_ref['id']) self.mox.ReplayAll() app = service.Service.create(host=host, binary=binary) @@ -101,15 +101,15 @@ class ServiceTestCase(test.BaseTestCase): host = 'foo' binary = 'bar' service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'report_count': 0, + 'id': 1} service.db.__getattr__('report_state') service.db.service_get_by_args(None, - host, - binary).AndReturn(service_ref) + host, + binary).AndReturn(service_ref) service.db.service_update(None, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) + mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() s = service.Service() @@ -119,22 +119,22 @@ class ServiceTestCase(test.BaseTestCase): host = 'foo' binary = 'bar' service_create = {'host': host, - 'binary': binary, - 'report_count': 0} + 'binary': binary, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'report_count': 0, + 'id': 1} service.db.__getattr__('report_state') service.db.service_get_by_args(None, host, binary).AndRaise(exception.NotFound()) service.db.service_create(None, - service_create).AndReturn(service_ref['id']) + service_create).AndReturn(service_ref['id']) service.db.service_get(None, service_ref['id']).AndReturn(service_ref) service.db.service_update(None, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) + mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() s = service.Service() @@ -144,14 +144,14 @@ class ServiceTestCase(test.BaseTestCase): host = 'foo' binary = 'bar' service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'report_count': 0, + 'id': 1} service.db.__getattr__('report_state') service.db.service_get_by_args(None, - host, - binary).AndRaise(Exception()) + host, + binary).AndRaise(Exception()) self.mox.ReplayAll() s = service.Service() @@ -163,16 +163,16 @@ class ServiceTestCase(test.BaseTestCase): host = 'foo' binary = 'bar' service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'report_count': 0, + 'id': 1} service.db.__getattr__('report_state') service.db.service_get_by_args(None, - host, - binary).AndReturn(service_ref) + host, + binary).AndReturn(service_ref) service.db.service_update(None, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) + mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() s = service.Service() diff --git a/nova/tests/storage_unittest.py b/nova/tests/storage_unittest.py deleted file mode 100644 index f400cd2f..00000000 --- a/nova/tests/storage_unittest.py +++ /dev/null @@ -1,115 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -from nova import exception -from nova import flags -from nova import test -from nova.compute import node -from nova.volume import storage - - -FLAGS = flags.FLAGS - - -class StorageTestCase(test.TrialTestCase): - def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) - super(StorageTestCase, self).setUp() - self.mynode = node.Node() - self.mystorage = None - self.flags(connection_type='fake', - fake_storage=True) - self.mystorage = storage.BlockStore() - - def test_run_create_volume(self): - vol_size = '0' - user_id = 'fake' - project_id = 'fake' - volume_id = self.mystorage.create_volume(vol_size, user_id, project_id) - # TODO(termie): get_volume returns differently than create_volume - self.assertEqual(volume_id, - storage.get_volume(volume_id)['volume_id']) - - rv = self.mystorage.delete_volume(volume_id) - self.assertRaises(exception.Error, - storage.get_volume, - volume_id) - - def test_too_big_volume(self): - vol_size = '1001' - user_id = 'fake' - project_id = 'fake' - self.assertRaises(TypeError, - self.mystorage.create_volume, - vol_size, user_id, project_id) - - def test_too_many_volumes(self): - vol_size = '1' - user_id = 'fake' - project_id = 'fake' - num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1 - total_slots = FLAGS.slots_per_shelf * num_shelves - vols = [] - for i in xrange(total_slots): - vid = self.mystorage.create_volume(vol_size, user_id, project_id) - vols.append(vid) - self.assertRaises(storage.NoMoreVolumes, - self.mystorage.create_volume, - vol_size, user_id, project_id) - for id in vols: - self.mystorage.delete_volume(id) - - def test_run_attach_detach_volume(self): - # Create one volume and one node to test with - instance_id = "storage-test" - vol_size = "5" - user_id = "fake" - project_id = 'fake' - mountpoint = "/dev/sdf" - volume_id = self.mystorage.create_volume(vol_size, user_id, project_id) - - volume_obj = storage.get_volume(volume_id) - volume_obj.start_attach(instance_id, mountpoint) - rv = yield self.mynode.attach_volume(volume_id, - instance_id, - mountpoint) - self.assertEqual(volume_obj['status'], "in-use") - self.assertEqual(volume_obj['attachStatus'], "attached") - self.assertEqual(volume_obj['instance_id'], instance_id) - self.assertEqual(volume_obj['mountpoint'], mountpoint) - - self.assertRaises(exception.Error, - self.mystorage.delete_volume, - volume_id) - - rv = yield self.mystorage.detach_volume(volume_id) - volume_obj = storage.get_volume(volume_id) - self.assertEqual(volume_obj['status'], "available") - - rv = self.mystorage.delete_volume(volume_id) - self.assertRaises(exception.Error, - storage.get_volume, - volume_id) - - def test_multi_node(self): - # TODO(termie): Figure out how to test with two nodes, - # each of them having a different FLAG for storage_node - # This will allow us to test cross-node interactions - pass diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 0df0c20d..99b22870 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -37,8 +37,7 @@ class VolumeTestCase(test.TrialTestCase): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() self.compute = utils.import_object(FLAGS.compute_manager) - self.flags(connection_type='fake', - fake_storage=True) + self.flags(connection_type='fake') self.volume = utils.import_object(FLAGS.volume_manager) self.context = None From 76b894b93e9ce45fd6cf8d89166c1f60b2a41311 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 21:45:50 -0700 Subject: [PATCH 084/113] fixed a few bugs in volume handling --- nova/tests/compute_unittest.py | 2 +- nova/tests/volume_unittest.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 07a2fceb..746c035d 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -60,7 +60,7 @@ class ComputeTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 - return db.instance_create(None, inst) + return db.instance_create(self.context, inst) @defer.inlineCallbacks def test_run_terminate(self): diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 99b22870..9e35d2a1 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -99,7 +99,16 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance""" - instance_id = "storage-test" + inst = {} + inst['image_id'] = 'ami-test' + inst['reservation_id'] = 'r-fakeres' + inst['launch_time'] = '10' + inst['user_id'] = 'fake' + inst['project_id'] = 'fake' + inst['instance_type'] = 'm1.tiny' + inst['mac_address'] = utils.generate_mac() + inst['ami_launch_index'] = 0 + instance_id = db.instance_create(self.context, inst) mountpoint = "/dev/sdf" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) @@ -112,8 +121,9 @@ class VolumeTestCase(test.TrialTestCase): vol = db.volume_get(None, volume_id) self.assertEqual(vol['status'], "in-use") self.assertEqual(vol['attach_status'], "attached") - self.assertEqual(vol['instance_id'], instance_id) self.assertEqual(vol['mountpoint'], mountpoint) + instance_ref = db.volume_get_instance(self.context, volume_id) + self.assertEqual(instance_ref['id'], instance_id) self.assertFailure(self.volume.delete_volume(self.context, volume_id), exception.Error) @@ -130,6 +140,7 @@ class VolumeTestCase(test.TrialTestCase): db.volume_get, None, volume_id) + db.instance_destroy(self.context, instance_id) @defer.inlineCallbacks def test_concurrent_volumes_get_different_blades(self): From d5a623727eeef60035e97b395b443a20f96de3b5 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 8 Sep 2010 10:45:39 +0200 Subject: [PATCH 086/113] Make the scripts in bin/ detect if they're being run from a bzr checkout or an extracted release tarball or whatever and adjust PYTHONPATH accordingly. --- bin/nova-api | 10 ++++++++++ bin/nova-api-new | 11 +++++++++++ bin/nova-compute | 11 +++++++++++ bin/nova-dhcpbridge | 11 +++++++---- bin/nova-import-canonical-imagestore | 8 ++++++++ bin/nova-instancemonitor | 10 ++++++++++ bin/nova-manage | 9 +++++++++ bin/nova-network | 11 +++++++++++ bin/nova-objectstore | 11 +++++++++++ bin/nova-volume | 11 +++++++++++ 10 files changed, 99 insertions(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a3ad5a0e..ede09d38 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -22,9 +22,19 @@ Tornado daemon for the main API endpoint. """ import logging +import os +import sys from tornado import httpserver from tornado import ioloop +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import flags from nova import server from nova import utils diff --git a/bin/nova-api-new b/bin/nova-api-new index fda42339..8625c487 100755 --- a/bin/nova-api-new +++ b/bin/nova-api-new @@ -21,6 +21,17 @@ Nova API daemon. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import api from nova import flags from nova import utils diff --git a/bin/nova-compute b/bin/nova-compute index ed9a5556..631d2c85 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -21,6 +21,17 @@ Twistd daemon for the nova compute nodes. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import twistd from nova.compute import service diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 1f2ed4f8..0c3d987a 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -25,10 +25,13 @@ import logging import os import sys -#TODO(joshua): there is concern that the user dnsmasq runs under will not -# have nova in the path. This should be verified and if it is -# not true the ugly line below can be removed -sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) from nova import flags from nova import rpc diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 2bc61cf0..4ed9e836 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -29,6 +29,14 @@ import subprocess import sys import urllib2 +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import flags from nova import utils from nova.objectstore import image diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index fbac5888..094da403 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -21,9 +21,19 @@ Daemon for Nova RRD based instance resource monitoring. """ +import os import logging +import sys from twisted.application import service +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import twistd from nova.compute import monitor diff --git a/bin/nova-manage b/bin/nova-manage index 145294d3..d2fd49d8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -22,9 +22,18 @@ Connects to the running ADMIN api in the api daemon. """ +import os import sys import time +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import flags from nova import utils from nova.auth import manager diff --git a/bin/nova-network b/bin/nova-network index 5753aafb..307795d7 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -21,6 +21,17 @@ Twistd daemon for the nova network nodes. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import flags from nova import twistd diff --git a/bin/nova-objectstore b/bin/nova-objectstore index afcf13e2..447ef905 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -21,6 +21,17 @@ Twisted daemon for nova objectstore. Supports S3 API. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import flags from nova import utils from nova import twistd diff --git a/bin/nova-volume b/bin/nova-volume index 8ef006eb..f8e2e174 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -21,6 +21,17 @@ Twistd daemon for the nova volume nodes. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import twistd from nova.volume import service From 1f671ecde14c010fbb267a61eba0a094452df1cf Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 8 Sep 2010 01:53:07 -0700 Subject: [PATCH 087/113] make timestamps for instances and volumes, includes additions to get deleted objects from db using deleted flag. --- nova/tests/compute_unittest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 746c035d..e5da6b05 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -18,6 +18,8 @@ """ Tests For Compute """ + +import datetime import logging from twisted.internet import defer @@ -79,6 +81,24 @@ class ComputeTestCase(test.TrialTestCase): logging.info("After terminating instances: %s", instances) self.assertEqual(len(instances), 0) + @defer.inlineCallbacks + def test_run_terminate_timestamps(self): + """Make sure it is possible to run and terminate instance""" + instance_id = self._create_instance() + instance_ref = db.instance_get(self.context, instance_id) + self.assertEqual(instance_ref['launched_at'], None) + self.assertEqual(instance_ref['terminated_at'], None) + launch = datetime.datetime.now() + yield self.compute.run_instance(self.context, instance_id) + instance_ref = db.instance_get(self.context, instance_id) + self.assert_(instance_ref['launched_at'] > launch) + self.assertEqual(instance_ref['terminated_at'], None) + terminate = datetime.datetime.now() + yield self.compute.terminate_instance(self.context, instance_id) + instance_ref = db.instance_get({'deleted': True}, instance_id) + self.assert_(instance_ref['launched_at'] < terminate) + self.assert_(instance_ref['terminated_at'] > terminate) + @defer.inlineCallbacks def test_reboot(self): """Ensure instance can be rebooted""" From 759fc8b287ff3c98c2d635273c2485fbf4dcdf38 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 06:06:29 -0700 Subject: [PATCH 088/113] switch to using utcnow --- nova/tests/compute_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index e5da6b05..8a7f7b64 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -88,12 +88,12 @@ class ComputeTestCase(test.TrialTestCase): instance_ref = db.instance_get(self.context, instance_id) self.assertEqual(instance_ref['launched_at'], None) self.assertEqual(instance_ref['terminated_at'], None) - launch = datetime.datetime.now() + launch = datetime.datetime.utcnow() yield self.compute.run_instance(self.context, instance_id) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] > launch) self.assertEqual(instance_ref['terminated_at'], None) - terminate = datetime.datetime.now() + terminate = datetime.datetime.utcnow() yield self.compute.terminate_instance(self.context, instance_id) instance_ref = db.instance_get({'deleted': True}, instance_id) self.assert_(instance_ref['launched_at'] < terminate) From 305e3b52e94df333e2d832afd8e0bdbdc6910188 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 06:55:54 -0700 Subject: [PATCH 089/113] speed up describe by loading fixed and floating ips --- nova/endpoint/cloud.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 709c967b..6958eacf 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -379,11 +379,11 @@ class CloudController(object): 'code': instance['state'], 'name': instance['state_description'] } - floating_addr = db.instance_get_floating_address(context, - instance['id']) + floating_addr = None + if instance['fixed_ip']['floating_ips']: + floating_addr = instance['fixed_ip']['floating_ips'][0]['str_id'] i['publicDnsName'] = floating_addr - fixed_addr = db.instance_get_fixed_address(context, - instance['id']) + fixed_addr = instance['fixed_ip']['str_id'] i['privateDnsName'] = fixed_addr if not i['publicDnsName']: i['publicDnsName'] = i['privateDnsName'] From cc957e2398a4859ae07ac83018fd811023cacb72 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 07:47:30 -0700 Subject: [PATCH 090/113] floating ip commands --- bin/nova-manage | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index ecef5d55..408a2d9c 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -26,6 +26,8 @@ import os import sys import time +import IPy + # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -218,12 +220,41 @@ class ProjectCommands(object): with open(filename, 'w') as f: f.write(zip_file) +class FloatingIpCommands(object): + """Class for managing floating ip.""" + + def create(self, host, range): + """Creates floating ips for host by range + arguments: host ip_range""" + for address in IPy.IP(range): + db.floating_ip_create(None, {'address': str(address), + 'host': host}) + + def delete(self, ip_range): + """Deletes floating ips by range + arguments: range""" + for address in IPy.IP(ip_range): + db.floating_ip_destroy(None, str(address)) + + + def list(self, host=None): + """Lists all floating ips (optionally by host) + arguments: [host]""" + if host == None: + floating_ips = db.floating_ip_get_all(None) + else: + floating_ips = db.floating_ip_get_all_by_host(None, host) + for floating_ip in floating_ips: + print "%s\t%s\ti-%s" % (floating_ip['host'], + floating_ip['address'], + floating_ip['instance_id']) CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), ('role', RoleCommands), ('vpn', VpnCommands), + ('floating', FloatingIpCommands) ] From 444b02f4e7ce1d79b04416766a99228dcd22199d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 07:53:57 -0700 Subject: [PATCH 091/113] list command for floating ips --- bin/nova-manage | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 408a2d9c..56191252 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -245,9 +245,12 @@ class FloatingIpCommands(object): else: floating_ips = db.floating_ip_get_all_by_host(None, host) for floating_ip in floating_ips: - print "%s\t%s\ti-%s" % (floating_ip['host'], - floating_ip['address'], - floating_ip['instance_id']) + instance = None + if floating_ip['fixed_ip']: + instance = floating_ip['fixed_ip']['instance']['str_id'] + print "%s\t%s\t%s" % (floating_ip['host'], + floating_ip['address'], + instance) CATEGORIES = [ ('user', UserCommands), From fd037cc02799b4f484fe739d08e00e8b2aed56fd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 09:59:55 -0700 Subject: [PATCH 092/113] fix volume delete issue and volume hostname display --- nova/endpoint/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 709c967b..c7355ccd 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -260,7 +260,7 @@ class CloudController(object): v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], volume['user_id'], - 'host', + volume['host'], volume['instance_id'], volume['mountpoint']) if volume['attach_status'] == 'attached': @@ -635,7 +635,7 @@ class CloudController(object): # TODO: return error if not authorized volume_ref = db.volume_get_by_str(context, volume_id) host = db.volume_get_host(context, volume_ref['id']) - rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), + rpc.cast(db.queue_get_for(context, FLAGS.volume_topic, host), {"method": "delete_volume", "args": {"context": None, "volume_id": volume_id}}) From f4616bfad8a74f70c764f1bdd3955633b7fe22ae Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 10:38:32 -0700 Subject: [PATCH 093/113] pass volume['id'] instead of string id to delete volume --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index c7355ccd..cb625bfa 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -638,7 +638,7 @@ class CloudController(object): rpc.cast(db.queue_get_for(context, FLAGS.volume_topic, host), {"method": "delete_volume", "args": {"context": None, - "volume_id": volume_id}}) + "volume_id": volume_ref['id']}}) return defer.succeed(True) @rbac.allow('all') From b97fb42e3f0c390eead4c508bdd3c8b4af889c25 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 10:43:19 -0700 Subject: [PATCH 094/113] remove extraneous get_host calls that were requiring an extra db trip --- nova/endpoint/cloud.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index cb625bfa..6cda7940 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -299,7 +299,7 @@ class CloudController(object): if volume_ref['attach_status'] == "attached": raise exception.ApiError("Volume is already attached") instance_ref = db.instance_get_by_str(context, instance_id) - host = db.instance_get_host(context, instance_ref['id']) + host = instance_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "attach_volume", "args": {"context": None, @@ -323,7 +323,7 @@ class CloudController(object): if volume_ref['status'] == "available": raise exception.Error("Volume is already detached") try: - host = db.instance_get_host(context, instance_ref['id']) + host = instance_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "detach_volume", "args": {"context": None, @@ -483,7 +483,7 @@ class CloudController(object): def _get_network_topic(self, context): """Retrieves the network host for a project""" network_ref = db.project_get_network(context, context.project.id) - host = db.network_get_host(context, network_ref['id']) + host = network_ref['host'] if not host: host = yield rpc.call(FLAGS.network_topic, {"method": "set_network_host", @@ -608,7 +608,7 @@ class CloudController(object): # we will need to cast here. db.fixed_ip_deallocate(context, address) - host = db.instance_get_host(context, instance_ref['id']) + host = instance_ref['host'] if host: rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "terminate_instance", @@ -623,7 +623,7 @@ class CloudController(object): """instance_id is a list of instance ids""" for id_str in instance_id: instance_ref = db.instance_get_by_str(context, id_str) - host = db.instance_get_host(context, instance_ref['id']) + host = instance_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "reboot_instance", "args": {"context": None, @@ -634,7 +634,7 @@ class CloudController(object): def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized volume_ref = db.volume_get_by_str(context, volume_id) - host = db.volume_get_host(context, volume_ref['id']) + host = volume_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.volume_topic, host), {"method": "delete_volume", "args": {"context": None, From a0cefa81004f87a58702d8b300bf65616b670371 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 11:02:37 -0700 Subject: [PATCH 095/113] fix describe addresses --- nova/endpoint/cloud.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 6cda7940..9a09454a 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -420,10 +420,12 @@ class CloudController(object): iterator = db.floating_ip_get_by_project(context, context.project.id) for floating_ip_ref in iterator: - address = floating_ip_ref['id_str'] - instance_ref = db.floating_ip_get_instance(address) + address = floating_ip_ref['str_id'] + instance_id = None + if floating_ip_ref['instance']: + instance_id = floating_ip_ref['instance']['str_id'] address_rv = {'public_ip': address, - 'instance_id': instance_ref['id_str']} + 'instance_id': instance_id} if context.user.is_admin(): details = "%s (%s)" % (address_rv['instance_id'], floating_ip_ref['project_id']) From 8988d72b329e110737e482ef607f4fda74b13242 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 11:07:10 -0700 Subject: [PATCH 096/113] solution that works with this version --- nova/endpoint/cloud.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 9a09454a..f84360c9 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -421,9 +421,8 @@ class CloudController(object): context.project.id) for floating_ip_ref in iterator: address = floating_ip_ref['str_id'] - instance_id = None - if floating_ip_ref['instance']: - instance_id = floating_ip_ref['instance']['str_id'] + instance_ref = db.floating_ip_get_instance(context, address) + instance_id = instance_ref['str_id'] address_rv = {'public_ip': address, 'instance_id': instance_id} if context.user.is_admin(): From bfe560870f94c04ef37c73b34bf75ac2dac36841 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 11:17:14 -0700 Subject: [PATCH 097/113] faster describe_addresses --- nova/endpoint/cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 2866474e..26bae065 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -421,8 +421,10 @@ class CloudController(object): context.project.id) for floating_ip_ref in iterator: address = floating_ip_ref['str_id'] - instance_ref = db.floating_ip_get_instance(context, address) - instance_id = instance_ref['str_id'] + instance_id = None + if (floating_ip_ref['fixed_ip'] + and floating_ip_ref['fixed_ip']['instance']): + instance_id = floating_ip_ref['fixed_ip']['instance']['str_id'] address_rv = {'public_ip': address, 'instance_id': instance_id} if context.user.is_admin(): From b7befd5d121eaad561382ff0f52eccc3a7f7a133 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 12:38:33 -0700 Subject: [PATCH 098/113] floating_address is the name for the cast --- nova/endpoint/cloud.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 2866474e..bb24d1f0 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -449,9 +449,9 @@ class CloudController(object): floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "deallocate_floating_ip", - "args": {"context": None, - "floating_ip": floating_ip_ref['str_id']}}) + {"method": "deallocate_floating_ip", + "args": {"context": None, + "floating_address": floating_ip_ref['str_id']}}) defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') @@ -462,11 +462,11 @@ class CloudController(object): floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "associate_floating_ip", - "args": {"context": None, - "floating_ip": floating_ip_ref['str_id'], - "fixed_ip": fixed_ip_ref['str_id'], - "instance_id": instance_ref['id']}}) + {"method": "associate_floating_ip", + "args": {"context": None, + "floating_address": floating_ip_ref['str_id'], + "fixed_address": fixed_ip_ref['str_id'], + "instance_id": instance_ref['id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') @@ -475,9 +475,9 @@ class CloudController(object): floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "disassociate_floating_ip", - "args": {"context": None, - "floating_ip": floating_ip_ref['str_id']}}) + {"method": "disassociate_floating_ip", + "args": {"context": None, + "floating_address": floating_ip_ref['str_id']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks @@ -487,9 +487,9 @@ class CloudController(object): host = network_ref['host'] if not host: host = yield rpc.call(FLAGS.network_topic, - {"method": "set_network_host", - "args": {"context": None, - "project_id": context.project.id}}) + {"method": "set_network_host", + "args": {"context": None, + "project_id": context.project.id}}) defer.returnValue(db.queue_get_for(context, FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') From fcb450d316beebdd610516fed6094df00c9c1d7a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 12:45:51 -0700 Subject: [PATCH 099/113] don't need to pass instance_id to network on associate --- nova/endpoint/cloud.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index bb24d1f0..397c9c55 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -465,8 +465,7 @@ class CloudController(object): {"method": "associate_floating_ip", "args": {"context": None, "floating_address": floating_ip_ref['str_id'], - "fixed_address": fixed_ip_ref['str_id'], - "instance_id": instance_ref['id']}}) + "fixed_address": fixed_ip_ref['str_id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') From d54cb543a3925dbd177baa732245367b1b4e3bc0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 21:52:06 -0700 Subject: [PATCH 100/113] make the db creates return refs instead of ids --- nova/endpoint/cloud.py | 2 +- nova/tests/compute_unittest.py | 2 +- nova/tests/network_unittest.py | 8 ++++---- nova/tests/service_unittest.py | 4 ++-- nova/tests/volume_unittest.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 397c9c55..7f4a901c 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -543,7 +543,7 @@ class CloudController(object): base_options['security_group'] = security_group for num in range(int(kwargs['max_count'])): - inst_id = db.instance_create(context, base_options) + inst_id = db.instance_create(context, base_options)['id'] inst = {} inst['mac_address'] = utils.generate_mac() diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 8a7f7b64..de2bf3d3 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -62,7 +62,7 @@ class ComputeTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 - return db.instance_create(self.context, inst) + return db.instance_create(self.context, inst)['id'] @defer.inlineCallbacks def test_run_terminate(self): diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index a89f1d62..9958600e 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -56,12 +56,12 @@ class NetworkTestCase(test.TrialTestCase): name)) # create the necessary network data for the project self.network.set_network_host(self.context, self.projects[i].id) - instance_id = db.instance_create(None, + instance_ref = db.instance_create(None, {'mac_address': utils.generate_mac()}) - self.instance_id = instance_id - instance_id = db.instance_create(None, + self.instance_id = instance_ref['id'] + instance_ref = db.instance_create(None, {'mac_address': utils.generate_mac()}) - self.instance2_id = instance_id + self.instance2_id = instance_ref['id'] def tearDown(self): # pylint: disable-msg=C0103 super(NetworkTestCase, self).tearDown() diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 097a045e..01da0eb8 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -87,7 +87,7 @@ class ServiceTestCase(test.BaseTestCase): host, binary).AndRaise(exception.NotFound()) service.db.service_create(None, - service_create).AndReturn(service_ref['id']) + service_create).AndReturn(service_ref) self.mox.ReplayAll() app = service.Service.create(host=host, binary=binary) @@ -131,7 +131,7 @@ class ServiceTestCase(test.BaseTestCase): host, binary).AndRaise(exception.NotFound()) service.db.service_create(None, - service_create).AndReturn(service_ref['id']) + service_create).AndReturn(service_ref) service.db.service_get(None, service_ref['id']).AndReturn(service_ref) service.db.service_update(None, service_ref['id'], mox.ContainsKeyValue('report_count', 1)) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 9e35d2a1..1d665b50 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -108,7 +108,7 @@ class VolumeTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 - instance_id = db.instance_create(self.context, inst) + instance_id = db.instance_create(self.context, inst)['id'] mountpoint = "/dev/sdf" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) From 3ecd942ee8c126cf852c50fc188ed2bfedf50c57 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 21:56:46 -0700 Subject: [PATCH 101/113] fix rare condition where describe is called before instance has an ip --- nova/endpoint/cloud.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 281c4535..5ff69edf 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -362,12 +362,14 @@ class CloudController(object): def _format_instances(self, context, reservation_id=None): reservations = {} if reservation_id: - instances = db.instance_get_by_reservation(context, reservation_id) + instances = db.instance_get_by_reservation(context, + reservation_id) else: if not context.user.is_admin(): instances = db.instance_get_all(context) else: - instances = db.instance_get_by_project(context, context.project.id) + instances = db.instance_get_by_project(context, + context.project.id) for instance in instances: if not context.user.is_admin(): if instance['image_id'] == FLAGS.vpn_image_id: @@ -379,12 +381,15 @@ class CloudController(object): 'code': instance['state'], 'name': instance['state_description'] } + fixed_addr = None floating_addr = None - if instance['fixed_ip']['floating_ips']: - floating_addr = instance['fixed_ip']['floating_ips'][0]['str_id'] - i['publicDnsName'] = floating_addr - fixed_addr = instance['fixed_ip']['str_id'] + if instance['fixed_ip']: + fixed_addr = instance['fixed_ip']['str_id'] + if instance['fixed_ip']['floating_ips']: + fixed = instance['fixed_ip'] + floating_addr = fixed['floating_ips'][0]['str_id'] i['privateDnsName'] = fixed_addr + i['publicDnsName'] = floating_addr if not i['publicDnsName']: i['publicDnsName'] = i['privateDnsName'] i['dnsName'] = None From 9a11274175519daae0f23442d5ff3788ce279cb9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 05:25:57 -0700 Subject: [PATCH 102/113] fixed messed up call in metadata --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 5ff69edf..6f8cf94f 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -84,7 +84,7 @@ class CloudController(object): def _get_mpi_data(self, project_id): result = {} - for instance in db.instance_get_by_project(project_id): + for instance in db.instance_get_by_project(None, project_id): line = '%s slots=%d' % (instance.fixed_ip['str_id'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: From 307a8085479eb9307cddd7c07783ee9660eaae8a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 05:38:59 -0700 Subject: [PATCH 103/113] typo in metadata call --- nova/endpoint/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 6f8cf94f..bf2f07ad 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -108,8 +108,8 @@ class CloudController(object): else: keys = '' hostname = instance_ref['hostname'] - floating_ip = db.instance_get_floating_ip_address(None, - instance_ref['id']) + floating_ip = db.instance_get_floating_address(None, + instance_ref['id']) data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { From 7aa05c5af3681fec90038942127ebf0b470f85d4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 05:49:36 -0700 Subject: [PATCH 104/113] couple more errors in metadata --- nova/endpoint/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index bf2f07ad..c85383ef 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -114,7 +114,7 @@ class CloudController(object): 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { 'ami-id': instance_ref['image_id'], - 'ami-launch-index': instance_ref['ami_launch_index'], + 'ami-launch-index': instance_ref['launch_index'], 'ami-manifest-path': 'FIXME', 'block-device-mapping': { # TODO(vish): replace with real data 'ami': 'sda1', @@ -130,7 +130,7 @@ class CloudController(object): 'local-ipv4': address, 'kernel-id': instance_ref['kernel_id'], 'placement': { - 'availaibility-zone': instance_ref['availability_zone'], + 'availability-zone': 'nova' # TODO(vish): real zone }, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', From 229fd1881ddee117367efc1dd45facd89131a0af Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 12:25:45 -0700 Subject: [PATCH 105/113] dhcpbridge needed host instead of node name --- bin/nova-dhcpbridge | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index c4795cca..42eaf4bc 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -53,7 +53,7 @@ def add_lease(_mac, ip_address, _hostname, _interface): network_manager = utils.import_object(FLAGS.network_manager) network_manager.lease_fixed_ip(None, ip_address) else: - rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.host), {"method": "lease_fixed_ip", "args": {"context": None, "address": ip_address}}) @@ -71,7 +71,7 @@ def del_lease(_mac, ip_address, _hostname, _interface): network_manager = utils.import_object(FLAGS.network_manager) network_manager.release_fixed_ip(None, ip_address) else: - rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.host), {"method": "release_fixed_ip", "args": {"context": None, "address": ip_address}}) From aaf9d445cd485223e38179b0a6aaa22118e97634 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 12:34:45 -0700 Subject: [PATCH 106/113] hostname should be string id --- nova/endpoint/cloud.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index c85383ef..2406e820 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -550,12 +550,13 @@ class CloudController(object): base_options['security_group'] = security_group for num in range(int(kwargs['max_count'])): - inst_id = db.instance_create(context, base_options)['id'] + instance_ref = db.instance_create(context, base_options) + inst_id = instance_ref['id'] inst = {} inst['mac_address'] = utils.generate_mac() inst['launch_index'] = num - inst['hostname'] = inst_id + inst['hostname'] = instance_ref['str_id'] db.instance_update(context, inst_id, inst) address = self.network_manager.allocate_fixed_ip(context, inst_id, From 96bfc8d8d97f7182d1d7dee0bd59c21e08434f00 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 14:16:14 -0700 Subject: [PATCH 107/113] fix mpi 500 on fixed ip --- nova/endpoint/cloud.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 2406e820..925d14e1 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -85,12 +85,13 @@ class CloudController(object): def _get_mpi_data(self, project_id): result = {} for instance in db.instance_get_by_project(None, project_id): - line = '%s slots=%d' % (instance.fixed_ip['str_id'], - INSTANCE_TYPES[instance['instance_type']]['vcpus']) - if instance['key_name'] in result: - result[instance['key_name']].append(line) - else: - result[instance['key_name']] = [line] + if instance['fixed_ip']: + line = '%s slots=%d' % (instance['fixed_ip']['str_id'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) + if instance['key_name'] in result: + result[instance['key_name']].append(line) + else: + result[instance['key_name']] = [line] return result def get_metadata(self, address): From e5a90a6de65b1d1bc1562df67d0699ea5215473a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 15:04:52 -0700 Subject: [PATCH 108/113] set dnsName on describe --- nova/endpoint/cloud.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 925d14e1..6ca6855c 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -391,9 +391,7 @@ class CloudController(object): floating_addr = fixed['floating_ips'][0]['str_id'] i['privateDnsName'] = fixed_addr i['publicDnsName'] = floating_addr - if not i['publicDnsName']: - i['publicDnsName'] = i['privateDnsName'] - i['dnsName'] = None + i['dnsName'] = i['publicDnsName'] or i['privateDnsName'] i['keyName'] = instance['key_name'] if context.user.is_admin(): i['keyName'] = '%s (%s, %s)' % (i['keyName'], From 961b20a77acd0cdc69bd57b5c66297a0d4421225 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 00:16:12 -0700 Subject: [PATCH 109/113] improved network error case handling for fixed ips --- bin/nova-dhcpbridge | 10 +++++---- nova/tests/network_unittest.py | 41 ++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 42eaf4bc..2f75bf43 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -46,16 +46,17 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') -def add_lease(_mac, ip_address, _hostname, _interface): +def add_lease(mac, ip_address, _hostname, _interface): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: logging.debug("leasing ip") network_manager = utils.import_object(FLAGS.network_manager) - network_manager.lease_fixed_ip(None, ip_address) + network_manager.lease_fixed_ip(None, mac, ip_address) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.host), {"method": "lease_fixed_ip", "args": {"context": None, + "mac": mac, "address": ip_address}}) @@ -64,16 +65,17 @@ def old_lease(_mac, _ip_address, _hostname, _interface): logging.debug("Adopted old lease or got a change of mac/hostname") -def del_lease(_mac, ip_address, _hostname, _interface): +def del_lease(mac, ip_address, _hostname, _interface): """Called when a lease expires.""" if FLAGS.fake_rabbit: logging.debug("releasing ip") network_manager = utils.import_object(FLAGS.network_manager) - network_manager.release_fixed_ip(None, ip_address) + network_manager.release_fixed_ip(None, mac, ip_address) else: rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.host), {"method": "release_fixed_ip", "args": {"context": None, + "mac": mac, "address": ip_address}}) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 9958600e..d8d4ec0c 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -147,10 +147,23 @@ class NetworkTestCase(test.TrialTestCase): """Makes sure that private ips don't overlap""" first = self._create_address(0) lease_ip(first) + instance_ids = [] for i in range(1, 5): - address = self._create_address(i) - address2 = self._create_address(i) - address3 = self._create_address(i) + mac = utils.generate_mac() + instance_ref = db.instance_create(None, + {'mac_address': mac}) + instance_ids.append(instance_ref['id']) + address = self._create_address(i, instance_ref['id']) + mac = utils.generate_mac() + instance_ref = db.instance_create(None, + {'mac_address': mac}) + instance_ids.append(instance_ref['id']) + address2 = self._create_address(i, instance_ref['id']) + mac = utils.generate_mac() + instance_ref = db.instance_create(None, + {'mac_address': mac}) + instance_ids.append(instance_ref['id']) + address3 = self._create_address(i, instance_ref['id']) lease_ip(address) lease_ip(address2) lease_ip(address3) @@ -166,6 +179,8 @@ class NetworkTestCase(test.TrialTestCase): release_ip(address) release_ip(address2) release_ip(address3) + for instance_id in instance_ids: + db.instance_destroy(None, instance_id) release_ip(first) db.fixed_ip_deallocate(None, first) @@ -226,8 +241,13 @@ class NetworkTestCase(test.TrialTestCase): num_available_ips = db.network_count_available_ips(None, network['id']) addresses = [] + instance_ids = [] for i in range(num_available_ips): - address = self._create_address(0) + mac = utils.generate_mac() + instance_ref = db.instance_create(None, + {'mac_address': mac}) + instance_ids.append(instance_ref['id']) + address = self._create_address(0, instance_ref['id']) addresses.append(address) lease_ip(address) @@ -238,9 +258,10 @@ class NetworkTestCase(test.TrialTestCase): None, network['id']) - for i in range(len(addresses)): + for i in range(num_available_ips): db.fixed_ip_deallocate(None, addresses[i]) release_ip(addresses[i]) + db.instance_destroy(None, instance_ids[i]) self.assertEqual(db.network_count_available_ips(None, network['id']), num_available_ips) @@ -263,7 +284,10 @@ def binpath(script): def lease_ip(private_ip): """Run add command on dhcpbridge""" network_ref = db.fixed_ip_get_network(None, private_ip) - cmd = "%s add fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) + instance_ref = db.fixed_ip_get_instance(None, private_ip) + cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), + instance_ref['mac_address'], + private_ip) env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} @@ -274,7 +298,10 @@ def lease_ip(private_ip): def release_ip(private_ip): """Run del command on dhcpbridge""" network_ref = db.fixed_ip_get_network(None, private_ip) - cmd = "%s del fake %s fake" % (binpath('nova-dhcpbridge'), private_ip) + instance_ref = db.fixed_ip_get_instance(None, private_ip) + cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), + instance_ref['mac_address'], + private_ip) env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} From c90d504306bfca045f1a0cfd1a182b722a67ac34 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 04:01:44 -0700 Subject: [PATCH 110/113] fixed tests, added a flag for updating dhcp on disassociate --- nova/endpoint/cloud.py | 2 +- nova/tests/network_unittest.py | 37 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 6ca6855c..622b4e2a 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -613,7 +613,7 @@ class CloudController(object): # NOTE(vish): Currently, nothing needs to be done on the # network node until release. If this changes, # we will need to cast here. - db.fixed_ip_deallocate(context, address) + self.network.deallocate_fixed_ip(context, address) host = instance_ref['host'] if host: diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index d8d4ec0c..dc5277f0 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -28,6 +28,7 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager +from nova.endpoint import api FLAGS = flags.FLAGS @@ -48,7 +49,7 @@ class NetworkTestCase(test.TrialTestCase): self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.network = utils.import_object(FLAGS.network_manager) - self.context = None + self.context = api.APIRequestContext(None, project=None, user=self.user) for i in range(5): name = 'project%s' % i self.projects.append(self.manager.create_project(name, @@ -75,12 +76,10 @@ class NetworkTestCase(test.TrialTestCase): def _create_address(self, project_num, instance_id=None): """Create an address in given project num""" - net = db.project_get_network(None, self.projects[project_num].id) - address = db.fixed_ip_allocate(None, net['id']) if instance_id is None: instance_id = self.instance_id - db.fixed_ip_instance_associate(None, address, instance_id) - return address + self.context.project = self.projects[project_num] + return self.network.allocate_fixed_ip(self.context, instance_id) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" @@ -103,14 +102,14 @@ class NetworkTestCase(test.TrialTestCase): address = db.instance_get_floating_address(None, self.instance_id) self.assertEqual(address, None) self.network.deallocate_floating_ip(self.context, float_addr) - db.fixed_ip_deallocate(None, fix_addr) + self.network.deallocate_fixed_ip(self.context, fix_addr) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" address = self._create_address(0) self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) lease_ip(address) - db.fixed_ip_deallocate(None, address) + self.network.deallocate_fixed_ip(self.context, address) # Doesn't go away until it's dhcp released self.assertTrue(is_allocated_in_project(address, self.projects[0].id)) @@ -131,14 +130,14 @@ class NetworkTestCase(test.TrialTestCase): lease_ip(address) lease_ip(address2) - db.fixed_ip_deallocate(None, address) + self.network.deallocate_fixed_ip(self.context, address) release_ip(address) self.assertFalse(is_allocated_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second self.assertTrue(is_allocated_in_project(address2, self.projects[1].id)) - db.fixed_ip_deallocate(None, address2) + self.network.deallocate_fixed_ip(self.context, address2) release_ip(address2) self.assertFalse(is_allocated_in_project(address2, self.projects[1].id)) @@ -173,16 +172,16 @@ class NetworkTestCase(test.TrialTestCase): self.projects[0].id)) self.assertFalse(is_allocated_in_project(address3, self.projects[0].id)) - db.fixed_ip_deallocate(None, address) - db.fixed_ip_deallocate(None, address2) - db.fixed_ip_deallocate(None, address3) + self.network.deallocate_fixed_ip(self.context, address) + self.network.deallocate_fixed_ip(self.context, address2) + self.network.deallocate_fixed_ip(self.context, address3) release_ip(address) release_ip(address2) release_ip(address3) for instance_id in instance_ids: db.instance_destroy(None, instance_id) release_ip(first) - db.fixed_ip_deallocate(None, first) + self.network.deallocate_fixed_ip(self.context, first) def test_vpn_ip_and_port_looks_valid(self): """Ensure the vpn ip and port are reasonable""" @@ -209,12 +208,12 @@ class NetworkTestCase(test.TrialTestCase): """Makes sure that ip addresses that are deallocated get reused""" address = self._create_address(0) lease_ip(address) - db.fixed_ip_deallocate(None, address) + self.network.deallocate_fixed_ip(self.context, address) release_ip(address) address2 = self._create_address(0) self.assertEqual(address, address2) - db.fixed_ip_deallocate(None, address2) + self.network.deallocate_fixed_ip(self.context, address2) def test_available_ips(self): """Make sure the number of available ips for the network is correct @@ -254,12 +253,12 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(db.network_count_available_ips(None, network['id']), 0) self.assertRaises(db.NoMoreAddresses, - db.fixed_ip_allocate, - None, - network['id']) + self.network.allocate_fixed_ip, + self.context, + 'foo') for i in range(num_available_ips): - db.fixed_ip_deallocate(None, addresses[i]) + self.network.deallocate_fixed_ip(self.context, addresses[i]) release_ip(addresses[i]) db.instance_destroy(None, instance_ids[i]) self.assertEqual(db.network_count_available_ips(None, From 24c846f1f1d99adb0a105ef32027f5e3203b3b16 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 04:06:22 -0700 Subject: [PATCH 111/113] typo fixes, add flag to nova-dhcpbridge --- bin/nova-dhcpbridge | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 2f75bf43..a127ed03 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -44,6 +44,7 @@ flags.DECLARE('auth_driver', 'nova.auth.manager') flags.DECLARE('redis_db', 'nova.datastore') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') +flags.DECLARE('update_dhcp_on_disassociate', 'nova.network.manager') def add_lease(mac, ip_address, _hostname, _interface): From d039d7a30bd6ac90c566f1890b288a7010ce79f8 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 13 Sep 2010 09:03:14 +0200 Subject: [PATCH 112/113] Move vol.destroy() call out of the _check method in test_multiple_volume_race_condition test and into a callback of the DeferredList. This should fix the intermittent failure of that test. I /think/ test_too_many_volumes's failure was caused by test_multiple_volume_race_condition failure, since I have not been able to reproduce its failure after fixing this one. --- nova/tests/volume_unittest.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 2a07afe6..540b7158 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -128,7 +128,6 @@ class VolumeTestCase(test.TrialTestCase): volume_service.get_volume, volume_id) - @defer.inlineCallbacks def test_multiple_volume_race_condition(self): vol_size = "5" user_id = "fake" @@ -137,17 +136,28 @@ class VolumeTestCase(test.TrialTestCase): def _check(volume_id): vol = volume_service.get_volume(volume_id) shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id']) - self.assert_(shelf_blade not in shelf_blades) + self.assertTrue(shelf_blade not in shelf_blades, + "Same shelf/blade tuple came back twice") shelf_blades.append(shelf_blade) logging.debug("got %s" % shelf_blade) - vol.destroy() + return vol deferreds = [] for i in range(5): d = self.volume.create_volume(vol_size, user_id, project_id) d.addCallback(_check) d.addErrback(self.fail) deferreds.append(d) - yield defer.DeferredList(deferreds) + def destroy_volumes(retvals): + overall_succes = True + for success, volume in retvals: + if not success: + overall_succes = False + else: + volume.destroy() + self.assertTrue(overall_succes) + d = defer.DeferredList(deferreds) + d.addCallback(destroy_volumes) + return d def test_multi_node(self): # TODO(termie): Figure out how to test with two nodes, From 48bacdd68accf7f4dfeca413c6ecc9fb499efd5b Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 13 Sep 2010 00:06:32 -0700 Subject: [PATCH 113/113] add a shell to nova-manage, which respects flags (taken from django) --- bin/nova-manage | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index d2fd49d8..6e526676 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -17,6 +17,37 @@ # License for the specific language governing permissions and limitations # under the License. +# Interactive shell based on Django: +# +# Copyright (c) 2005, the Lawrence Journal-World +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of Django nor the names of its contributors may be used +# to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + """ CLI interface for nova management. Connects to the running ADMIN api in the api daemon. @@ -103,6 +134,29 @@ class VpnCommands(object): self.pipe.launch_vpn_instance(project_id) +class ShellCommands(object): + def run(self): + "Runs a Python interactive interpreter. Tries to use IPython, if it's available." + try: + import IPython + # Explicitly pass an empty list as arguments, because otherwise IPython + # would use sys.argv from this script. + shell = IPython.Shell.IPShell(argv=[]) + shell.mainloop() + except ImportError: + import code + try: # Try activating rlcompleter, because it's handy. + import readline + except ImportError: + pass + else: + # We don't have to wrap the following import in a 'try', because + # we already know 'readline' was imported successfully. + import rlcompleter + readline.parse_and_bind("tab:complete") + code.interact() + + class RoleCommands(object): """Class for managing roles.""" @@ -225,6 +279,7 @@ CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), ('role', RoleCommands), + ('shell', ShellCommands), ('vpn', VpnCommands), ]