From 47ec3fa70e6cda4eecc165c189a735db424c45b4 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 3 Dec 2010 00:01:21 +0000 Subject: [PATCH 01/54] * Removes unused schema * Removes MUST uid from novaUser * Changes isAdmin to isNovaAdmin * Adds two new configuration options: ** ldap_user_id_attribute, with a default of uid ** ldap_user_name_attribute, with a default of cn * ldapdriver.py has been modified to use these changes Rationale: Removing uid from novaUser: Requiring uid makes the schema very posix specific. Other schemas don't use uid for identifiers at all. This change makes the schema more interoperable. Changing isAdmin to isNovaAdmin: This attribute is too generic. It doesn't describe what the user is an admin of, and in a pre-existing directory is out of place. This change is to make the attribute more specific to the software. Adding config options for id and name: This is another interoperability change. This change makes the driver more compatible with directories like AD, where sAMAccountName is used instead of uid. Also, some directory admins prefer to use displayName rather than CN for full names of users. --- nova/auth/ldapdriver.py | 21 ++++++++++++--------- nova/auth/nova_openldap.schema | 26 +++----------------------- nova/auth/nova_sun.schema | 6 ++---- nova/auth/openssh-lpk_openldap.schema | 19 ------------------- nova/auth/openssh-lpk_sun.schema | 10 ---------- 5 files changed, 17 insertions(+), 65 deletions(-) delete mode 100644 nova/auth/openssh-lpk_openldap.schema delete mode 100644 nova/auth/openssh-lpk_sun.schema diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d6..e4c36c28 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -37,6 +37,8 @@ flags.DEFINE_string('ldap_url', 'ldap://localhost', flags.DEFINE_string('ldap_password', 'changeme', 'LDAP password') flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', 'DN of admin user') +flags.DEFINE_string('ldap_user_id_attribute', 'uid', 'Attribute to use as id') +flags.DEFINE_string('ldap_user_name_attribute', 'cn', 'Attribute to use as name') flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') @@ -131,12 +133,12 @@ class LdapDriver(object): 'inetOrgPerson', 'novaUser']), ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), + (FLAGS.ldap_user_id_attribute, [name]), ('sn', [name]), - ('cn', [name]), + (FLAGS.ldap_user_name_attribute, [name]), ('secretKey', [secret_key]), ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), + ('isNovaAdmin', [str(is_admin).upper()]), ] self.conn.add_s(self.__uid_to_dn(name), attr) return self.__to_user(dict(attr)) @@ -274,7 +276,7 @@ class LdapDriver(object): if secret_key: attr.append((self.ldap.MOD_REPLACE, 'secretKey', secret_key)) if admin is not None: - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', str(admin).upper())) + attr.append((self.ldap.MOD_REPLACE, 'isNovaAdmin', str(admin).upper())) self.conn.modify_s(self.__uid_to_dn(uid), attr) def __user_exists(self, uid): @@ -450,11 +452,11 @@ class LdapDriver(object): if attr == None: return None return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], + 'id': attr[FLAGS.ldap_user_id_attribute][0], + 'name': attr[FLAGS.ldap_user_name_attribute][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} + 'admin': (attr['isNovaAdmin'][0] == 'TRUE')} def __to_project(self, attr): """Convert ldap attributes to Project object""" @@ -474,9 +476,10 @@ class LdapDriver(object): return dn.split(',')[0].split('=')[1] @staticmethod - def __uid_to_dn(dn): + def __uid_to_dn(uid): """Convert uid to dn""" - return 'uid=%s,%s' % (dn, FLAGS.ldap_user_subtree) + return FLAGS.ldap_user_id_attribute + '=%s,%s' \ + % (uid, FLAGS.ldap_user_subtree) class FakeLdapDriver(LdapDriver): diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema index 4047361d..9e528f58 100644 --- a/nova/auth/nova_openldap.schema +++ b/nova/auth/nova_openldap.schema @@ -30,20 +30,10 @@ attributetype ( SINGLE-VALUE ) -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - attributetype ( novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' + NAME 'isNovaAdmin' + DESC 'Is user an nova administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE @@ -61,17 +51,7 @@ objectClass ( NAME 'novaUser' DESC 'access and secret keys' AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) + MAY ( accessKey $ secretKey $ isNovaAdmin ) ) objectClass ( diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema index e925e05e..decf10f0 100644 --- a/nova/auth/nova_sun.schema +++ b/nova/auth/nova_sun.schema @@ -8,9 +8,7 @@ dn: cn=schema attributeTypes: ( 1.3.6.1.3.1.666.666.3.1 NAME 'accessKey' DESC 'Key for accessing data' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) attributeTypes: ( 1.3.6.1.3.1.666.666.3.2 NAME 'secretKey' DESC 'Secret key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.3 NAME 'keyFingerprint' DESC 'Fingerprint of private key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isAdmin' DESC 'Is user an administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isNovaAdmin' DESC 'Is user a nova administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) attributeTypes: ( 1.3.6.1.3.1.666.666.3.5 NAME 'projectManager' DESC 'Project Managers of a project' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MUST ( uid ) MAY ( accessKey $ secretKey $ isAdmin ) ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.2 NAME 'novaKeyPair' DESC 'Key pair for User' SUP top STRUCTURAL MUST ( cn $ sshPublicKey $ keyFingerprint ) ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MAY ( accessKey $ secretKey $ isNovaAdmin ) ) objectClasses: ( 1.3.6.1.3.1.666.666.4.3 NAME 'novaProject' DESC 'Container for project' SUP groupOfNames STRUCTURAL MUST ( cn $ projectManager ) ) diff --git a/nova/auth/openssh-lpk_openldap.schema b/nova/auth/openssh-lpk_openldap.schema deleted file mode 100644 index 93351da6..00000000 --- a/nova/auth/openssh-lpk_openldap.schema +++ /dev/null @@ -1,19 +0,0 @@ -# -# LDAP Public Key Patch schema for use with openssh-ldappubkey -# Author: Eric AUGE -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema deleted file mode 100644 index 5f52db3b..00000000 --- a/nova/auth/openssh-lpk_sun.schema +++ /dev/null @@ -1,10 +0,0 @@ -# -# LDAP Public Key Patch schema for use with openssh-ldappubkey -# Author: Eric AUGE -# -# Schema for Sun Directory Server. -# Based on the original schema, modified by Stefan Fischer. -# -dn: cn=schema -attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) -objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) From fc4776319f9cd7e1c89a4210af2401eb416c0f4b Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 10:22:29 +0000 Subject: [PATCH 02/54] Removing novaProject from the schema. This change may look odd at first; here's how it works: Both roles are projects are groupOfNames. Previously, we were differentiating projects from project roles by using the novaProject objectclass on the project, and not on the roles. This change removes novaProject, and uses the owner attribute instead of the projectManager attribute. Only projects should have an owner. We can differentiate projects from project roles by checking for the existence of this attribute. To check for the existence of an attribute in LDAP, a wildcard search is used. The fake LDAP driver did not support wildcard searches, so I put in "all or nothing" support for it. The wildcard search support doesn't work exactly like wildcard searches in LDAP, but will work for the case that's required. --- nova/auth/fakeldap.py | 3 +++ nova/auth/ldapdriver.py | 16 ++++++++-------- nova/auth/nova_openldap.schema | 16 ---------------- nova/auth/nova_sun.schema | 2 -- nova/auth/opendj.sh | 2 -- nova/auth/slap.sh | 4 +--- 6 files changed, 12 insertions(+), 31 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 46e0135b..2dcb6926 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -119,6 +119,9 @@ def _match(key, value, attrs): """Match a given key and value against an attribute list.""" if key not in attrs: return False + # This is a wild card search. Implemented as all or nothing for now. + if value == "*": + return True if key != "objectclass": return value in attrs[key] # it is an objectclass check, so check subclasses diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 87151566..705e89ee 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -106,7 +106,7 @@ class LdapDriver(object): """Retrieve project by id""" dn = 'cn=%s,%s' % (pid, FLAGS.ldap_project_subtree) - attr = self.__find_object(dn, '(objectclass=novaProject)') + attr = self.__find_object(dn, '(owner=*)') return self.__to_project(attr) def get_users(self): @@ -122,7 +122,7 @@ class LdapDriver(object): def get_projects(self, uid=None): """Retrieve list of projects""" - pattern = '(objectclass=novaProject)' + pattern = '(owner=*)' if uid: pattern = "(&%s(member=%s))" % (pattern, self.__uid_to_dn(uid)) attrs = self.__find_objects(FLAGS.ldap_project_subtree, @@ -205,10 +205,10 @@ class LdapDriver(object): if not manager_dn in members: members.append(manager_dn) attr = [ - ('objectclass', ['novaProject']), + ('objectclass', ['groupOfNames']), ('cn', [name]), ('description', [description]), - ('projectManager', [manager_dn]), + ('owner', [manager_dn]), ('member', members)] self.conn.add_s('cn=%s,%s' % (name, FLAGS.ldap_project_subtree), attr) return self.__to_project(dict(attr)) @@ -224,7 +224,7 @@ class LdapDriver(object): "manager %s doesn't exist" % manager_uid) manager_dn = self.__uid_to_dn(manager_uid) - attr.append((self.ldap.MOD_REPLACE, 'projectManager', manager_dn)) + attr.append((self.ldap.MOD_REPLACE, 'owner', manager_dn)) if description: attr.append((self.ldap.MOD_REPLACE, 'description', description)) self.conn.modify_s('cn=%s,%s' % (project_id, @@ -286,7 +286,7 @@ class LdapDriver(object): project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) roles = self.__find_objects(project_dn, '(&(&(objectclass=groupOfNames)' - '(!(objectclass=novaProject)))' + '(!(owner=*)))' '(member=%s))' % self.__uid_to_dn(uid)) return [role['cn'][0] for role in roles] @@ -385,7 +385,7 @@ class LdapDriver(object): def __find_role_dns(self, tree): """Find dns of role objects in given tree""" return self.__find_dns(tree, - '(&(objectclass=groupOfNames)(!(objectclass=novaProject)))') + '(&(objectclass=groupOfNames)(!(owner=*)))') def __find_group_dns_with_member(self, tree, uid): """Find dns of group objects in a given tree that contain member""" @@ -534,7 +534,7 @@ class LdapDriver(object): return { 'id': attr['cn'][0], 'name': attr['cn'][0], - 'project_manager_id': self.__dn_to_uid(attr['projectManager'][0]), + 'project_manager_id': self.__dn_to_uid(attr['owner'][0]), 'description': attr.get('description', [None])[0], 'member_ids': [self.__dn_to_uid(x) for x in member_dns]} diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema index 9e528f58..1a10a445 100644 --- a/nova/auth/nova_openldap.schema +++ b/nova/auth/nova_openldap.schema @@ -39,13 +39,6 @@ attributetype ( SINGLE-VALUE ) -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - objectClass ( novaOCs:1 NAME 'novaUser' @@ -53,12 +46,3 @@ objectClass ( AUXILIARY MAY ( accessKey $ secretKey $ isNovaAdmin ) ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema index decf10f0..1a04601b 100644 --- a/nova/auth/nova_sun.schema +++ b/nova/auth/nova_sun.schema @@ -9,6 +9,4 @@ dn: cn=schema attributeTypes: ( 1.3.6.1.3.1.666.666.3.1 NAME 'accessKey' DESC 'Key for accessing data' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) attributeTypes: ( 1.3.6.1.3.1.666.666.3.2 NAME 'secretKey' DESC 'Secret key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isNovaAdmin' DESC 'Is user a nova administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.5 NAME 'projectManager' DESC 'Project Managers of a project' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MAY ( accessKey $ secretKey $ isNovaAdmin ) ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.3 NAME 'novaProject' DESC 'Container for project' SUP groupOfNames STRUCTURAL MUST ( cn $ projectManager ) ) diff --git a/nova/auth/opendj.sh b/nova/auth/opendj.sh index 8052c077..9a960034 100755 --- a/nova/auth/opendj.sh +++ b/nova/auth/opendj.sh @@ -30,9 +30,7 @@ fi abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` schemapath='/var/opendj/instance/config/schema' -cp $abspath/openssh-lpk_sun.schema $schemapath/97-openssh-lpk_sun.ldif cp $abspath/nova_sun.schema $schemapath/98-nova_sun.ldif -chown opendj:opendj $schemapath/97-openssh-lpk_sun.ldif chown opendj:opendj $schemapath/98-nova_sun.ldif cat >/etc/ldap/ldap.conf </dev/null; echo "$PWD"/"${0##*/}")"` -cp $abspath/openssh-lpk_openldap.schema /etc/ldap/schema/openssh-lpk_openldap.schema -cp $abspath/nova_openldap.schema /etc/ldap/schema/nova_openldap.schema +cp $abspath/nova_openldap.schema /etc/ldap/schema/nova.schema mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig cat >/etc/ldap/slapd.conf </etc/ldap/slapd.conf < Date: Wed, 8 Dec 2010 16:23:59 +0000 Subject: [PATCH 03/54] Adding support for choosing a schema version, so that users can more easily migrate from an old schema to the new schema. --- nova/auth/ldapdriver.py | 79 ++++++++++++++++++++-------------- nova/auth/nova_openldap.schema | 4 +- nova/auth/nova_sun.schema | 5 ++- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 705e89ee..21d8f806 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -32,6 +32,8 @@ from nova import flags FLAGS = flags.FLAGS +flags.DEFINE_integer('ldap_schema_version', 1, + 'Current version of the LDAP schema') flags.DEFINE_string('ldap_url', 'ldap://localhost', 'Point this at your ldap server') flags.DEFINE_string('ldap_password', 'changeme', 'LDAP password') @@ -75,10 +77,20 @@ class LdapDriver(object): Defines enter and exit and therefore supports the with/as syntax. """ + project_pattern = '(owner=*)' + isadmin_attribute = 'isNovaAdmin' + project_attribute = 'owner' + project_objectclass = 'groupOfNames' + def __init__(self): """Imports the LDAP module""" self.ldap = __import__('ldap') self.conn = None + if FLAGS.ldap_schema_version == 1: + LdapDriver.project_pattern = '(objectclass=novaProject)' + LdapDriver.isadmin_attribute = 'isAdmin' + LdapDriver.project_attribute = 'projectManager' + LdapDriver.project_objectclass = 'novaProject' def __enter__(self): """Creates the connection to LDAP""" @@ -106,7 +118,7 @@ class LdapDriver(object): """Retrieve project by id""" dn = 'cn=%s,%s' % (pid, FLAGS.ldap_project_subtree) - attr = self.__find_object(dn, '(owner=*)') + attr = self.__find_object(dn, LdapDriver.project_pattern) return self.__to_project(attr) def get_users(self): @@ -122,7 +134,7 @@ class LdapDriver(object): def get_projects(self, uid=None): """Retrieve list of projects""" - pattern = '(owner=*)' + pattern = LdapDriver.project_pattern if uid: pattern = "(&%s(member=%s))" % (pattern, self.__uid_to_dn(uid)) attrs = self.__find_objects(FLAGS.ldap_project_subtree, @@ -152,11 +164,11 @@ class LdapDriver(object): else: attr.append((self.ldap.MOD_ADD, 'accessKey', \ [access_key])) - if 'isNovaAdmin' in user.keys(): - attr.append((self.ldap.MOD_REPLACE, 'isNovaAdmin', \ + if LdapDriver.isadmin_attribute in user.keys(): + attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, \ [str(is_admin).upper()])) else: - attr.append((self.ldap.MOD_ADD, 'isNovaAdmin', \ + attr.append((self.ldap.MOD_ADD, LdapDriver.isadmin_attribute, \ [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) @@ -175,7 +187,7 @@ class LdapDriver(object): (FLAGS.ldap_user_name_attribute, [name]), ('secretKey', [secret_key]), ('accessKey', [access_key]), - ('isNovaAdmin', [str(is_admin).upper()]), + (LdapDriver.isadmin_attribute, [str(is_admin).upper()]), ] self.conn.add_s(self.__uid_to_dn(name), attr) return self.__to_user(dict(attr)) @@ -205,10 +217,10 @@ class LdapDriver(object): if not manager_dn in members: members.append(manager_dn) attr = [ - ('objectclass', ['groupOfNames']), + ('objectclass', [LdapDriver.project_objectclass]), ('cn', [name]), ('description', [description]), - ('owner', [manager_dn]), + (LdapDriver.project_attribute, [manager_dn]), ('member', members)] self.conn.add_s('cn=%s,%s' % (name, FLAGS.ldap_project_subtree), attr) return self.__to_project(dict(attr)) @@ -224,7 +236,7 @@ class LdapDriver(object): "manager %s doesn't exist" % manager_uid) manager_dn = self.__uid_to_dn(manager_uid) - attr.append((self.ldap.MOD_REPLACE, 'owner', manager_dn)) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.project_attribute, manager_dn)) if description: attr.append((self.ldap.MOD_REPLACE, 'description', description)) self.conn.modify_s('cn=%s,%s' % (project_id, @@ -284,10 +296,9 @@ class LdapDriver(object): return roles else: project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) - roles = self.__find_objects(project_dn, - '(&(&(objectclass=groupOfNames)' - '(!(owner=*)))' - '(member=%s))' % self.__uid_to_dn(uid)) + query = ('(&(&(objectclass=groupOfNames)(!%s))(member=%s))' % + (LdapDriver.project_pattern, self.__uid_to_dn(uid))) + roles = self.__find_objects(project_dn, query) return [role['cn'][0] for role in roles] def delete_user(self, uid): @@ -306,9 +317,9 @@ class LdapDriver(object): if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'accessKey', \ user['accessKey'])) - if 'isNovaAdmin' in user.keys(): - attr.append((self.ldap.MOD_DELETE, 'isNovaAdmin', \ - user['isNovaAdmin'])) + if LdapDriver.isadmin_attribute in user.keys(): + attr.append((self.ldap.MOD_DELETE, LdapDriver.isadmin_attribute, \ + user[LdapDriver.isadmin_attribute])) self.conn.modify_s(self.__uid_to_dn(uid), attr) else: # Delete entry @@ -330,7 +341,7 @@ class LdapDriver(object): if secret_key: attr.append((self.ldap.MOD_REPLACE, 'secretKey', secret_key)) if admin is not None: - attr.append((self.ldap.MOD_REPLACE, 'isNovaAdmin', str(admin).upper())) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, str(admin).upper())) self.conn.modify_s(self.__uid_to_dn(uid), attr) def __user_exists(self, uid): @@ -384,19 +395,20 @@ class LdapDriver(object): def __find_role_dns(self, tree): """Find dns of role objects in given tree""" - return self.__find_dns(tree, - '(&(objectclass=groupOfNames)(!(owner=*)))') + query = '(&(objectclass=groupOfNames)(!%s))' % LdapDriver.project_pattern + return self.__find_dns(tree, query) def __find_group_dns_with_member(self, tree, uid): """Find dns of group objects in a given tree that contain member""" - dns = self.__find_dns(tree, - '(&(objectclass=groupOfNames)(member=%s))' % - self.__uid_to_dn(uid)) + query = ('(&(objectclass=groupOfNames)(member=%s))' % + self.__uid_to_dn(uid)) + dns = self.__find_dns(tree, query) return dns def __group_exists(self, dn): """Check if group exists""" - return self.__find_object(dn, '(objectclass=groupOfNames)') is not None + query = '(objectclass=groupOfNames)' + return self.__find_object(dn, query) is not None @staticmethod def __role_to_dn(role, project_id=None): @@ -435,7 +447,7 @@ class LdapDriver(object): """Check if user is in group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be searched in group " - "becuase the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): return False res = self.__find_object(group_dn, @@ -447,10 +459,10 @@ class LdapDriver(object): """Add user to group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be added to the group " - "becuase the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): raise exception.NotFound("The group at dn %s doesn't exist" % - (group_dn,)) + group_dn) if self.__is_in_group(uid, group_dn): raise exception.Duplicate("User %s is already a member of " "the group %s" % (uid, group_dn)) @@ -461,13 +473,13 @@ class LdapDriver(object): """Remove user from group""" if not self.__group_exists(group_dn): raise exception.NotFound("The group at dn %s doesn't exist" % - (group_dn,)) + group_dn) if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from the " - "group because the user doesn't exist" % (uid,)) + "group because the user doesn't exist" % uid) if not self.__is_in_group(uid, group_dn): raise exception.NotFound("User %s is not a member of the group" % - (uid,)) + uid) # NOTE(vish): remove user from group and any sub_groups sub_dns = self.__find_group_dns_with_member( group_dn, uid) @@ -489,7 +501,7 @@ class LdapDriver(object): """Remove user from all roles and projects""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from all " - "because the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) role_dns = self.__find_group_dns_with_member( FLAGS.role_project_subtree, uid) for role_dn in role_dns: @@ -516,13 +528,13 @@ class LdapDriver(object): if attr is None: return None if ('accessKey' in attr.keys() and 'secretKey' in attr.keys() \ - and 'isNovaAdmin' in attr.keys()): + and LdapDriver.isadmin_attribute in attr.keys()): return { 'id': attr[FLAGS.ldap_user_id_attribute][0], 'name': attr[FLAGS.ldap_user_name_attribute][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], - 'admin': (attr['isNovaAdmin'][0] == 'TRUE')} + 'admin': (attr[LdapDriver.isadmin_attribute][0] == 'TRUE')} else: return None @@ -534,7 +546,8 @@ class LdapDriver(object): return { 'id': attr['cn'][0], 'name': attr['cn'][0], - 'project_manager_id': self.__dn_to_uid(attr['owner'][0]), + 'project_manager_id': + self.__dn_to_uid(attr[LdapDriver.project_attribute][0]), 'description': attr.get('description', [None])[0], 'member_ids': [self.__dn_to_uid(x) for x in member_dns]} diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema index 1a10a445..daa3a844 100644 --- a/nova/auth/nova_openldap.schema +++ b/nova/auth/nova_openldap.schema @@ -1,7 +1,9 @@ # # Person object for Nova # inetorgperson with extra attributes -# Author: Vishvananda Ishaya +# Schema version: 2 +# Authors: Vishvananda Ishaya +# Ryan Lane # # diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema index 1a04601b..8e9052de 100644 --- a/nova/auth/nova_sun.schema +++ b/nova/auth/nova_sun.schema @@ -1,8 +1,9 @@ # # Person object for Nova # inetorgperson with extra attributes -# Author: Vishvananda Ishaya -# Modified for strict RFC 4512 compatibility by: Ryan Lane +# Schema version: 2 +# Authors: Vishvananda Ishaya +# Ryan Lane # # using internet experimental oid arc as per BP64 3.1 dn: cn=schema From e39d984a88eb9796de0a834a3a17a2c2fd683db8 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 16:26:12 +0000 Subject: [PATCH 04/54] Setting the default schema version to the new schema --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 21d8f806..eac1db54 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -32,7 +32,7 @@ from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_integer('ldap_schema_version', 1, +flags.DEFINE_integer('ldap_schema_version', 2, 'Current version of the LDAP schema') flags.DEFINE_string('ldap_url', 'ldap://localhost', 'Point this at your ldap server') From 2a4c32b57cf73dda36fc9d74dd4c99f227a0ba1e Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 16:38:35 +0000 Subject: [PATCH 05/54] PEP8 fixes --- nova/auth/ldapdriver.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index eac1db54..870262a1 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,7 +40,8 @@ flags.DEFINE_string('ldap_password', 'changeme', 'LDAP password') flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', 'DN of admin user') flags.DEFINE_string('ldap_user_id_attribute', 'uid', 'Attribute to use as id') -flags.DEFINE_string('ldap_user_name_attribute', 'cn', 'Attribute to use as name') +flags.DEFINE_string('ldap_user_name_attribute', 'cn', + 'Attribute to use as name') flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') @@ -153,23 +154,23 @@ class LdapDriver(object): # Malformed entries are useless, replace attributes found. attr = [] if 'secretKey' in user.keys(): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ + attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) else: - attr.append((self.ldap.MOD_ADD, 'secretKey', \ + attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) if 'accessKey' in user.keys(): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ + attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) else: - attr.append((self.ldap.MOD_ADD, 'accessKey', \ + attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) if LdapDriver.isadmin_attribute in user.keys(): - attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, \ - [str(is_admin).upper()])) + attr.append((self.ldap.MOD_REPLACE, + LdapDriver.isadmin_attribute, [str(is_admin).upper()])) else: - attr.append((self.ldap.MOD_ADD, LdapDriver.isadmin_attribute, \ - [str(is_admin).upper()])) + attr.append((self.ldap.MOD_ADD, + LdapDriver.isadmin_attribute, [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: @@ -236,7 +237,8 @@ class LdapDriver(object): "manager %s doesn't exist" % manager_uid) manager_dn = self.__uid_to_dn(manager_uid) - attr.append((self.ldap.MOD_REPLACE, LdapDriver.project_attribute, manager_dn)) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.project_attribute, + manager_dn)) if description: attr.append((self.ldap.MOD_REPLACE, 'description', description)) self.conn.modify_s('cn=%s,%s' % (project_id, @@ -312,14 +314,15 @@ class LdapDriver(object): # Retrieve user by name user = self.__get_ldap_user(uid) if 'secretKey' in user.keys(): - attr.append((self.ldap.MOD_DELETE, 'secretKey', \ - user['secretKey'])) + attr.append((self.ldap.MOD_DELETE, 'secretKey', + user['secretKey'])) if 'accessKey' in user.keys(): - attr.append((self.ldap.MOD_DELETE, 'accessKey', \ - user['accessKey'])) + attr.append((self.ldap.MOD_DELETE, 'accessKey', + user['accessKey'])) if LdapDriver.isadmin_attribute in user.keys(): - attr.append((self.ldap.MOD_DELETE, LdapDriver.isadmin_attribute, \ - user[LdapDriver.isadmin_attribute])) + attr.append((self.ldap.MOD_DELETE, + LdapDriver.isadmin_attribute, + user[LdapDriver.isadmin_attribute])) self.conn.modify_s(self.__uid_to_dn(uid), attr) else: # Delete entry @@ -341,7 +344,8 @@ class LdapDriver(object): if secret_key: attr.append((self.ldap.MOD_REPLACE, 'secretKey', secret_key)) if admin is not None: - attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, str(admin).upper())) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, + str(admin).upper())) self.conn.modify_s(self.__uid_to_dn(uid), attr) def __user_exists(self, uid): @@ -395,7 +399,8 @@ class LdapDriver(object): def __find_role_dns(self, tree): """Find dns of role objects in given tree""" - query = '(&(objectclass=groupOfNames)(!%s))' % LdapDriver.project_pattern + query = ('(&(objectclass=groupOfNames)(!%s))' % + LdapDriver.project_pattern) return self.__find_dns(tree, query) def __find_group_dns_with_member(self, tree, uid): From 4a4bb6364ec966dd4e89359542095170945c1be2 Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Thu, 9 Dec 2010 16:42:52 +0000 Subject: [PATCH 06/54] raw instances can now be launched in xenapi (only as hvm at the moment) --- nova/flags.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 1f94feb0..1cb846c1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -234,6 +234,13 @@ DEFINE_string('default_ramdisk', 'ari-11111', 'default ramdisk to use, testing only') DEFINE_string('default_instance_type', 'm1.small', 'default instance type to use, testing only') +DEFINE_string('null_kernel', 'aki-00000000', + 'kernel image that indicates not to use a kernel, ' + ' but to use a raw disk image instead') +DEFINE_string('null_kernel_pv', 'aki-00000001', + 'kernel image that indicates not to use a kernel, ' + ' but to use a raw disk image for a paravirtualized guest') + DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server') DEFINE_string('vpn_key_suffix', From 8444d0e5824f85701d7c819ed409bc66e4df5191 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 10 Dec 2010 18:49:54 +0000 Subject: [PATCH 07/54] Format fixes and modification of Vish's email address. --- nova/auth/ldapdriver.py | 41 ++++++++++++++++++---------------- nova/auth/nova_openldap.schema | 2 +- nova/auth/nova_sun.schema | 2 +- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 870262a1..1b928e7d 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -125,7 +125,7 @@ class LdapDriver(object): def get_users(self): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, - '(objectclass=novaUser)') + '(objectclass=novaUser)') users = [] for attr in attrs: user = self.__to_user(attr) @@ -155,22 +155,24 @@ class LdapDriver(object): attr = [] if 'secretKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'secretKey', - [secret_key])) + [secret_key])) else: attr.append((self.ldap.MOD_ADD, 'secretKey', - [secret_key])) + [secret_key])) if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'accessKey', - [access_key])) + [access_key])) else: attr.append((self.ldap.MOD_ADD, 'accessKey', - [access_key])) + [access_key])) if LdapDriver.isadmin_attribute in user.keys(): attr.append((self.ldap.MOD_REPLACE, - LdapDriver.isadmin_attribute, [str(is_admin).upper()])) + LdapDriver.isadmin_attribute, + [str(is_admin).upper()])) else: attr.append((self.ldap.MOD_ADD, - LdapDriver.isadmin_attribute, [str(is_admin).upper()])) + LdapDriver.isadmin_attribute, + [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: @@ -299,7 +301,7 @@ class LdapDriver(object): else: project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) query = ('(&(&(objectclass=groupOfNames)(!%s))(member=%s))' % - (LdapDriver.project_pattern, self.__uid_to_dn(uid))) + (LdapDriver.project_pattern, self.__uid_to_dn(uid))) roles = self.__find_objects(project_dn, query) return [role['cn'][0] for role in roles] @@ -363,7 +365,7 @@ class LdapDriver(object): def __get_ldap_user(self, uid): """Retrieve LDAP user entry by id""" attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + '(objectclass=novaUser)') return attr def __find_object(self, dn, query=None, scope=None): @@ -406,7 +408,7 @@ class LdapDriver(object): def __find_group_dns_with_member(self, tree, uid): """Find dns of group objects in a given tree that contain member""" query = ('(&(objectclass=groupOfNames)(member=%s))' % - self.__uid_to_dn(uid)) + self.__uid_to_dn(uid)) dns = self.__find_dns(tree, query) return dns @@ -436,7 +438,8 @@ class LdapDriver(object): for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Group can't be created " - "because user %s doesn't exist" % member_uid) + "because user %s doesn't exist" % + member_uid) members.append(self.__uid_to_dn(member_uid)) dn = self.__uid_to_dn(uid) if not dn in members: @@ -452,7 +455,7 @@ class LdapDriver(object): """Check if user is in group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be searched in group " - "because the user doesn't exist" % uid) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): return False res = self.__find_object(group_dn, @@ -464,7 +467,7 @@ class LdapDriver(object): """Add user to group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be added to the group " - "because the user doesn't exist" % uid) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): raise exception.NotFound("The group at dn %s doesn't exist" % group_dn) @@ -481,13 +484,13 @@ class LdapDriver(object): group_dn) if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from the " - "group because the user doesn't exist" % uid) + "group because the user doesn't exist" % + uid) if not self.__is_in_group(uid, group_dn): raise exception.NotFound("User %s is not a member of the group" % uid) # NOTE(vish): remove user from group and any sub_groups - sub_dns = self.__find_group_dns_with_member( - group_dn, uid) + sub_dns = self.__find_group_dns_with_member(group_dn, uid) for sub_dn in sub_dns: self.__safe_remove_from_group(uid, sub_dn) @@ -506,7 +509,7 @@ class LdapDriver(object): """Remove user from all roles and projects""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from all " - "because the user doesn't exist" % uid) + "because the user doesn't exist" % uid) role_dns = self.__find_group_dns_with_member( FLAGS.role_project_subtree, uid) for role_dn in role_dns: @@ -564,8 +567,8 @@ class LdapDriver(object): @staticmethod def __uid_to_dn(uid): """Convert uid to dn""" - return FLAGS.ldap_user_id_attribute + '=%s,%s' \ - % (uid, FLAGS.ldap_user_subtree) + return (FLAGS.ldap_user_id_attribute + '=%s,%s' + % (uid, FLAGS.ldap_user_subtree)) class FakeLdapDriver(LdapDriver): diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema index daa3a844..539a5c42 100644 --- a/nova/auth/nova_openldap.schema +++ b/nova/auth/nova_openldap.schema @@ -2,7 +2,7 @@ # Person object for Nova # inetorgperson with extra attributes # Schema version: 2 -# Authors: Vishvananda Ishaya +# Authors: Vishvananda Ishaya # Ryan Lane # # diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema index 8e9052de..4a6a7883 100644 --- a/nova/auth/nova_sun.schema +++ b/nova/auth/nova_sun.schema @@ -2,7 +2,7 @@ # Person object for Nova # inetorgperson with extra attributes # Schema version: 2 -# Authors: Vishvananda Ishaya +# Authors: Vishvananda Ishaya # Ryan Lane # # using internet experimental oid arc as per BP64 3.1 From ff997b57b24b6a2cbcf55dcca25a2cebbfe2c7d2 Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Mon, 13 Dec 2010 16:11:12 +0000 Subject: [PATCH 08/54] only needs work for distinguishing pv from hvm --- .project | 17 +++++++++++++++++ .pydevproject | 10 ++++++++++ cuba-ubuntu.16501 | 0 nova-api.pid | 1 + nova-api.pid.lock | 0 nova-manage.conf | 26 ++++++++++++++++++++++++++ nova-network.pid | 1 + nova-objectstore.pid | 1 + nova-scheduler.pid | 1 + 9 files changed, 57 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 cuba-ubuntu.16501 create mode 100644 nova-api.pid create mode 100644 nova-api.pid.lock create mode 100644 nova-manage.conf create mode 100644 nova-network.pid create mode 100644 nova-objectstore.pid create mode 100644 nova-scheduler.pid diff --git a/.project b/.project new file mode 100644 index 00000000..1213c940 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Openstack-xs_raw_disk_images + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 00000000..8030d8e4 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,10 @@ + + + + +Default +python 2.6 + +/home/salvatore/xenapi + + diff --git a/cuba-ubuntu.16501 b/cuba-ubuntu.16501 new file mode 100644 index 00000000..e69de29b diff --git a/nova-api.pid b/nova-api.pid new file mode 100644 index 00000000..9bcd7ef1 --- /dev/null +++ b/nova-api.pid @@ -0,0 +1 @@ +16501 diff --git a/nova-api.pid.lock b/nova-api.pid.lock new file mode 100644 index 00000000..e69de29b diff --git a/nova-manage.conf b/nova-manage.conf new file mode 100644 index 00000000..6942b8cc --- /dev/null +++ b/nova-manage.conf @@ -0,0 +1,26 @@ +--verbose +--nodaemon +--FAKE_subdomain=ec2 +--ca_path=/home/salvatore/openstack/nova/CA +--keys_path=/home/salvatore/openstack/nova/keys +--networks_path=/home/salvatore/openstack/nova/networks +--images_path=/home/salvatore/openstack/nova/images +--buckets_path=/home/salvatore/openstack/nova/buckets +--logdir=/home/salvatore/openstack/nova +#--libvirt_type=qemu +--connection_type=xenapi +--xenapi_connection_url=http://camctxlabs-svr05.cam.eu.citrix.com +--xenapi_connection_password=_car0pet_ +--auth_driver=nova.auth.dbdriver.DbDriver +--sql_connection=mysql://root:nova@10.70.177.19/nova +--s3_host=10.70.177.19 +--network_manager=nova.network.manager.FlatManager +#--network_manager=nova.network.manager.VlanManager +--fixed_range=192.168.1.240/28 +--routing_source_ip=10.70.177.19 +--flat_network_bridge=xenbr0 +--flat_network_gateway=192.168.1.1 +--flat_network_netmask=255.255.255.0 +--flat_network_network=192.168.1.0 +--flat_network_ips=192.168.1.220,192.168.1.221,192.168.1.222 +--flat_network_broadcast=192.168.1.255 diff --git a/nova-network.pid b/nova-network.pid new file mode 100644 index 00000000..b81b1082 --- /dev/null +++ b/nova-network.pid @@ -0,0 +1 @@ +16479 \ No newline at end of file diff --git a/nova-objectstore.pid b/nova-objectstore.pid new file mode 100644 index 00000000..0967315f --- /dev/null +++ b/nova-objectstore.pid @@ -0,0 +1 @@ +16469 \ No newline at end of file diff --git a/nova-scheduler.pid b/nova-scheduler.pid new file mode 100644 index 00000000..d6bb441a --- /dev/null +++ b/nova-scheduler.pid @@ -0,0 +1 @@ +16490 \ No newline at end of file From 2a751f29a5e4b1146e5be9f2ff8f2144d85a8790 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 13 Dec 2010 18:22:56 +0000 Subject: [PATCH 09/54] second round for unit testing framework --- nova/tests/virt_unittest.py | 65 +++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index ba3fba83..61102263 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -30,6 +30,9 @@ from nova.virt.xenapi import volume_utils FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +# Those are XenAPI related +flags.DECLARE('target_host', 'nova.virt.xenapi_conn') +FLAGS.target_host = '127.0.0.1' class LibvirtConnTestCase(test.TrialTestCase): @@ -270,36 +273,62 @@ class XenAPIVolumeTestCase(test.TrialTestCase): self.helper = volume_utils.VolumeHelper self.helper.late_import() + def _create_volume(self, size='0'): + """Create a volume object.""" + vol = {} + vol['size'] = size + vol['user_id'] = 'fake' + vol['project_id'] = 'fake' + vol['host'] = 'localhost' + vol['availability_zone'] = FLAGS.storage_availability_zone + vol['status'] = "creating" + vol['attach_status'] = "detached" + return db.volume_create(context.get_admin_context(), vol) + def test_create_iscsi_storage_raise_no_exception(self): - info = self.helper.parse_volume_info(None, None) - label = 'SR-' - description = '' - self.helper.create_iscsi_storage_blocking(self.session, - info, - label, - description) + vol = self._create_volume() + info = yield self.helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') + label = None # For testing new SRs + description = 'Test-SR' + self.session.fail_next_call = False + sr_ref = self.helper.create_iscsi_storage_blocking(self.session, + info, + label, + description) + self.assertEqual(sr_ref, self.session.SR.FAKE_REF) + db.volume_destroy(context.get_admin_context(), vol['id']) def test_create_iscsi_storage_raise_unable_to_create_sr_exception(self): - info = self.helper.parse_volume_info(None, None) - label = None + vol = self._create_volume() + info = yield self.helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') + label = None # For testing new SRs description = None - self.assertFailure(self.helper.create_iscsi_storage_blocking(self.session, - info, - label, - description), - StorageError) + self.session.fail_next_call = True + self.assertRaises(volume_utils.StorageError, + self.helper.create_iscsi_storage_blocking, + self.session, + info, + label, + description) def test_find_sr_from_vbd_raise_no_exception(self): - pass + sr_ref = yield self.helper.find_sr_from_vbd(self.session, + self.session.VBD.FAKE_REF) + self.assertEqual(sr_ref, self.session.SR.FAKE_REF) - def test_destroy_iscsi_storage_raise_no_exception(self): + def test_destroy_iscsi_storage(self): pass def test_introduce_vdi_raise_no_exception(self): - pass + sr_ref = self.session.SR.FAKE_REF + self.helper.introduce_vdi_blocking(self.session, sr_ref) def test_introduce_vdi_raise_unable_get_vdi_record_exception(self): - pass + sr_ref = self.session.SR.FAKE_REF + self.session.fail_next_call = True + self.assertRaises(volume_utils.StorageError, + self.helper.introduce_vdi_blocking, + self.session, sr_ref) def tearDown(self): super(XenAPIVolumeTestCase, self).tearDown() From 478fb2844eaf0f04c578fd4d109b712797f9251a Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 13 Dec 2010 18:43:24 +0000 Subject: [PATCH 10/54] removing imports that should have not been there --- nova/tests/virt_unittest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index 61102263..1095662c 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -317,7 +317,8 @@ class XenAPIVolumeTestCase(test.TrialTestCase): self.assertEqual(sr_ref, self.session.SR.FAKE_REF) def test_destroy_iscsi_storage(self): - pass + sr_ref = self.session.SR.FAKE_REF + self.helper.destroy_iscsi_storage_blocking(self.session, sr_ref) def test_introduce_vdi_raise_no_exception(self): sr_ref = self.session.SR.FAKE_REF From 9ec2c03b2220fa07f6ed7297d35712a47b4b791b Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 13 Dec 2010 20:31:33 +0000 Subject: [PATCH 11/54] moving xenapi unittests changes into another branch --- nova/tests/virt_unittest.py | 76 ------------------------------------- 1 file changed, 76 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index 1095662c..d49383fb 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -25,14 +25,9 @@ from nova import utils from nova.api.ec2 import cloud from nova.auth import manager from nova.virt import libvirt_conn -from nova.virt.xenapi import fake -from nova.virt.xenapi import volume_utils FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -# Those are XenAPI related -flags.DECLARE('target_host', 'nova.virt.xenapi_conn') -FLAGS.target_host = '127.0.0.1' class LibvirtConnTestCase(test.TrialTestCase): @@ -262,74 +257,3 @@ class NWFilterTestCase(test.TrialTestCase): d.addCallback(lambda _: self.teardown_security_group()) return d - - -class XenAPIVolumeTestCase(test.TrialTestCase): - - def setUp(self): - super(XenAPIVolumeTestCase, self).setUp() - self.flags(xenapi_use_fake_session=True) - self.session = fake.FakeXenAPISession() - self.helper = volume_utils.VolumeHelper - self.helper.late_import() - - def _create_volume(self, size='0'): - """Create a volume object.""" - vol = {} - vol['size'] = size - vol['user_id'] = 'fake' - vol['project_id'] = 'fake' - vol['host'] = 'localhost' - vol['availability_zone'] = FLAGS.storage_availability_zone - vol['status'] = "creating" - vol['attach_status'] = "detached" - return db.volume_create(context.get_admin_context(), vol) - - def test_create_iscsi_storage_raise_no_exception(self): - vol = self._create_volume() - info = yield self.helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') - label = None # For testing new SRs - description = 'Test-SR' - self.session.fail_next_call = False - sr_ref = self.helper.create_iscsi_storage_blocking(self.session, - info, - label, - description) - self.assertEqual(sr_ref, self.session.SR.FAKE_REF) - db.volume_destroy(context.get_admin_context(), vol['id']) - - def test_create_iscsi_storage_raise_unable_to_create_sr_exception(self): - vol = self._create_volume() - info = yield self.helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') - label = None # For testing new SRs - description = None - self.session.fail_next_call = True - self.assertRaises(volume_utils.StorageError, - self.helper.create_iscsi_storage_blocking, - self.session, - info, - label, - description) - - def test_find_sr_from_vbd_raise_no_exception(self): - sr_ref = yield self.helper.find_sr_from_vbd(self.session, - self.session.VBD.FAKE_REF) - self.assertEqual(sr_ref, self.session.SR.FAKE_REF) - - def test_destroy_iscsi_storage(self): - sr_ref = self.session.SR.FAKE_REF - self.helper.destroy_iscsi_storage_blocking(self.session, sr_ref) - - def test_introduce_vdi_raise_no_exception(self): - sr_ref = self.session.SR.FAKE_REF - self.helper.introduce_vdi_blocking(self.session, sr_ref) - - def test_introduce_vdi_raise_unable_get_vdi_record_exception(self): - sr_ref = self.session.SR.FAKE_REF - self.session.fail_next_call = True - self.assertRaises(volume_utils.StorageError, - self.helper.introduce_vdi_blocking, - self.session, sr_ref) - - def tearDown(self): - super(XenAPIVolumeTestCase, self).tearDown() From dbdbef407987cc52de3e34dbcbc9cad90a95aa03 Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 15 Dec 2010 14:27:07 +0000 Subject: [PATCH 12/54] removed temporary comment lines --- nova/flags.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 1cb846c1..bd2fe4b9 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -237,10 +237,6 @@ DEFINE_string('default_instance_type', 'm1.small', DEFINE_string('null_kernel', 'aki-00000000', 'kernel image that indicates not to use a kernel, ' ' but to use a raw disk image instead') -DEFINE_string('null_kernel_pv', 'aki-00000001', - 'kernel image that indicates not to use a kernel, ' - ' but to use a raw disk image for a paravirtualized guest') - DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server') DEFINE_string('vpn_key_suffix', From cad11d8ad02121df6375f0a15fb39ca30925098d Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Wed, 15 Dec 2010 17:50:05 +0000 Subject: [PATCH 13/54] * code cleanup * revised unittest approach * added stubout and a number of tests --- nova/tests/xenapi_unittest.py | 136 +++++++++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 10 deletions(-) diff --git a/nova/tests/xenapi_unittest.py b/nova/tests/xenapi_unittest.py index c9be1732..b9955a94 100644 --- a/nova/tests/xenapi_unittest.py +++ b/nova/tests/xenapi_unittest.py @@ -31,7 +31,7 @@ # under the License. -import mox +import stubout import uuid from twisted.internet import defer @@ -51,20 +51,34 @@ from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils from nova.virt.xenapi import vm_utils from nova.virt.xenapi import volumeops +from boto.ec2.volume import Volume FLAGS = flags.FLAGS +def stubout_session(stubs, cls): + def fake_import(self): + fake_module = 'nova.virt.xenapi.fake' + from_list = ['fake'] + return __import__(fake_module, globals(), locals(), from_list, -1) + + stubs.Set(xenapi_conn.XenAPISession, '_create_session', + lambda s, url: cls(url)) + stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi', + fake_import) + + class XenAPIVolumeTestCase(test.TrialTestCase): """ - This uses Ewan's fake session approach + Unit tests for VM operations """ def setUp(self): super(XenAPIVolumeTestCase, self).setUp() - FLAGS.xenapi_use_fake_session = True + self.stubs = stubout.StubOutForTesting() FLAGS.target_host = '127.0.0.1' FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' + fake.reset() def _create_volume(self, size='0'): """Create a volume object.""" @@ -78,11 +92,12 @@ class XenAPIVolumeTestCase(test.TrialTestCase): vol['attach_status'] = "detached" return db.volume_create(context.get_admin_context(), vol) - def test_create_iscsi_storage_raise_no_exception(self): - fake.reset() + def test_create_iscsi_storage(self): + """ This shows how to test helper classes' methods """ + stubout_session(self.stubs, FakeSessionForVolumeTests) session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') helper = volume_utils.VolumeHelper - helper.late_import(FLAGS) + helper.XenAPI = session.get_imported_xenapi() vol = self._create_volume() info = yield helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') label = 'SR-%s' % vol['ec2_id'] @@ -93,8 +108,25 @@ class XenAPIVolumeTestCase(test.TrialTestCase): description) db.volume_destroy(context.get_admin_context(), vol['id']) + def test_parse_volume_info_raise_exception(self): + """ This shows how to test helper classes' methods """ + stubout_session(self.stubs, FakeSessionForVolumeTests) + session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') + helper = volume_utils.VolumeHelper + helper.XenAPI = session.get_imported_xenapi() + vol = self._create_volume() + # oops, wrong mount point! + info = helper.parse_volume_info(vol['ec2_id'], '/dev/sd') + + def check(exc): + self.assertIsInstance(exc.value, volume_utils.StorageError) + + info.addErrback(check) + db.volume_destroy(context.get_admin_context(), vol['id']) + def test_attach_volume(self): - fake.reset() + """ This shows how to test Ops classes' methods """ + stubout_session(self.stubs, FakeSessionForVolumeTests) conn = xenapi_conn.get_connection(False) volume = self._create_volume() instance = FakeInstance(1, 'fake', 'fake', 1, 2, 3, @@ -116,13 +148,34 @@ class XenAPIVolumeTestCase(test.TrialTestCase): result.addCallback(check) return result + def test_attach_volume_raise_exception(self): + """ This shows how to test when exceptions are raised """ + stubout_session(self.stubs, FakeSessionForVolumeFailedTests) + conn = xenapi_conn.get_connection(False) + volume = self._create_volume() + instance = FakeInstance(1, 'fake', 'fake', 1, 2, 3, + 'm1.large', 'aa:bb:cc:dd:ee:ff') + fake.create_vm(instance.name, 'Running') + result = conn.attach_volume(instance.name, volume['ec2_id'], + '/dev/sdc') + + def check(exc): + if exc: + pass + else: + self.fail('Oops, no exception has been raised!') + + result.addErrback(check) + return result + def tearDown(self): super(XenAPIVolumeTestCase, self).tearDown() + self.stubs.UnsetAll() class XenAPIVMTestCase(test.TrialTestCase): """ - This uses Ewan's fake session approach + Unit tests for VM operations """ def setUp(self): super(XenAPIVMTestCase, self).setUp() @@ -131,19 +184,20 @@ class XenAPIVMTestCase(test.TrialTestCase): admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) - FLAGS.xenapi_use_fake_session = True + self.stubs = stubout.StubOutForTesting() FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' fake.reset() fake.create_network('fake', FLAGS.flat_network_bridge) def test_list_instances_0(self): + stubout_session(self.stubs, FakeSessionForVMTests) conn = xenapi_conn.get_connection(False) instances = conn.list_instances() self.assertEquals(instances, []) - #test_list_instances_0.skip = "E" def test_spawn(self): + stubout_session(self.stubs, FakeSessionForVMTests) conn = xenapi_conn.get_connection(False) instance = FakeInstance(1, self.project.id, self.user.id, 1, 2, 3, 'm1.large', 'aa:bb:cc:dd:ee:ff') @@ -186,6 +240,7 @@ class XenAPIVMTestCase(test.TrialTestCase): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.stubs.UnsetAll() class FakeInstance(): @@ -199,3 +254,64 @@ class FakeInstance(): self.ramdisk_id = ramdisk_id self.instance_type = instance_type self.mac_address = mac_address + + +class FakeSessionForVMTests(fake.SessionBase): + def __init__(self, uri): + super(FakeSessionForVMTests, self).__init__(uri) + + def network_get_all_records_where(self, _1, _2): + return self.xenapi.network.get_all_records() + + def host_call_plugin(self, _1, _2, _3, _4, _5): + return '' + + def VM_start(self, _1, ref, _2, _3): + vm = fake.get_record('VM', ref) + if vm['power_state'] != 'Halted': + raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted', + vm['power_state']]) + vm['power_state'] = 'Running' + + +class FakeSessionForVolumeTests(fake.SessionBase): + def __init__(self, uri): + super(FakeSessionForVolumeTests, self).__init__(uri) + + def VBD_plug(self, _1, _2): + #FIXME(armando):make proper plug + pass + + def PBD_unplug(self, _1, _2): + #FIXME(armando):make proper unplug + pass + + def SR_forget(self, _1, _2): + #FIXME(armando):make proper forget + pass + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + #FIXME(armando):make proper introduce + valid_vdi = False + refs = fake.get_all('VDI') + for ref in refs: + rec = fake.get_record('VDI', ref) + if rec['uuid'] == uuid: + valid_vdi = True + if not valid_vdi: + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + +class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): + def __init__(self, uri): + super(FakeSessionForVolumeFailedTests, self).__init__(uri) + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + # test failure + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + def VBD_plug(self, _1, _2): + # test failure + raise fake.Failure([['INVALID_VBD', 'session', self._session]]) From 0534153594ace6d3c0080c26dfb4ee0486977212 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 15 Dec 2010 18:28:00 +0000 Subject: [PATCH 14/54] Adding back in openssh-lpk schema, as keys will likely be stored in LDAP again. --- nova/auth/opendj.sh | 1 + nova/auth/openssh-lpk_openldap.schema | 19 +++++++++++++++++++ nova/auth/openssh-lpk_sun.schema | 10 ++++++++++ nova/auth/slap.sh | 1 + 4 files changed, 31 insertions(+) create mode 100644 nova/auth/openssh-lpk_openldap.schema create mode 100644 nova/auth/openssh-lpk_sun.schema diff --git a/nova/auth/opendj.sh b/nova/auth/opendj.sh index 9a960034..1a280e5a 100755 --- a/nova/auth/opendj.sh +++ b/nova/auth/opendj.sh @@ -30,6 +30,7 @@ fi abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` schemapath='/var/opendj/instance/config/schema' +cp $abspath/openssh-lpk_sun.schema $schemapath/97-openssh-lpk_sun.ldif cp $abspath/nova_sun.schema $schemapath/98-nova_sun.ldif chown opendj:opendj $schemapath/98-nova_sun.ldif diff --git a/nova/auth/openssh-lpk_openldap.schema b/nova/auth/openssh-lpk_openldap.schema new file mode 100644 index 00000000..93351da6 --- /dev/null +++ b/nova/auth/openssh-lpk_openldap.schema @@ -0,0 +1,19 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Based on the proposal of : Mark Ruijter +# + + +# octetString SYNTAX +attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' + DESC 'MANDATORY: OpenSSH Public key' + EQUALITY octetStringMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) + +# printableString SYNTAX yes|no +objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY + DESC 'MANDATORY: OpenSSH LPK objectclass' + MAY ( sshPublicKey $ uid ) + ) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema new file mode 100644 index 00000000..5f52db3b --- /dev/null +++ b/nova/auth/openssh-lpk_sun.schema @@ -0,0 +1,10 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Schema for Sun Directory Server. +# Based on the original schema, modified by Stefan Fischer. +# +dn: cn=schema +attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) +objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) diff --git a/nova/auth/slap.sh b/nova/auth/slap.sh index 36c4ba37..95c61daf 100755 --- a/nova/auth/slap.sh +++ b/nova/auth/slap.sh @@ -21,6 +21,7 @@ apt-get install -y slapd ldap-utils python-ldap abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +cp $abspath/openssh-lpk_openldap.schema /etc/ldap/schema/openssh-lpk_openldap.schema cp $abspath/nova_openldap.schema /etc/ldap/schema/nova.schema mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig From 81a6e265c309af0bcc35cbf16889b0fd8ff569cb Mon Sep 17 00:00:00 2001 From: David Pravec Date: Thu, 16 Dec 2010 12:35:46 +0100 Subject: [PATCH 15/54] Make nova work even when user has LANG or LC_ALL configured Some commands are having different results when used in another language environment. For example ifconfig output parsing fails in my language. Also unittest using cat failed, as it didnt expect czech language in the error message. This small patch makes it work. Also adding myself to 'Authors' file. --- Authors | 1 + nova/tests/process_unittest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Authors b/Authors index 565444ee..6e0a7735 100644 --- a/Authors +++ b/Authors @@ -4,6 +4,7 @@ Anthony Young Armando Migliaccio Chris Behrens Chmouel Boudjnah +David Pravec Dean Troyer Devin Carlen Eldar Nugaev diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py index 67245af0..516b2452 100644 --- a/nova/tests/process_unittest.py +++ b/nova/tests/process_unittest.py @@ -49,7 +49,7 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stderr(self): pool = process.ProcessPool(2) - d = pool.simple_execute('cat BAD_FILE', check_exit_code=False) + d = pool.simple_execute('LC_ALL=C cat BAD_FILE', check_exit_code=False) def _check(rv): self.assertEqual(rv[0], '') From 331d83b5a2f01a35cda49bb976ce6f0efd851a9e Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 16 Dec 2010 16:33:38 +0000 Subject: [PATCH 16/54] Removed FakeInstance and introduced stubout for DB. Code clean-up --- nova/tests/xenapi/__init__.py | 33 +++++++ nova/tests/xenapi/stubs.py | 98 ++++++++++++++++++++ nova/tests/xenapi_unittest.py | 165 +++++++++------------------------- 3 files changed, 173 insertions(+), 123 deletions(-) create mode 100644 nova/tests/xenapi/__init__.py create mode 100644 nova/tests/xenapi/stubs.py diff --git a/nova/tests/xenapi/__init__.py b/nova/tests/xenapi/__init__.py new file mode 100644 index 00000000..a157d759 --- /dev/null +++ b/nova/tests/xenapi/__init__.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Stubouts, mocks and fixtures for the test suite""" diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py new file mode 100644 index 00000000..52518938 --- /dev/null +++ b/nova/tests/xenapi/stubs.py @@ -0,0 +1,98 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Stubouts, mocks and fixtures for the test suite""" + +from nova.virt import xenapi_conn +from nova.virt.xenapi import fake + + +def stubout_session(stubs, cls): + """ Stubs out two methods from XenAPISession """ + def fake_import(self): + """ Stubs out get_imported_xenapi of XenAPISession """ + fake_module = 'nova.virt.xenapi.fake' + from_list = ['fake'] + return __import__(fake_module, globals(), locals(), from_list, -1) + + stubs.Set(xenapi_conn.XenAPISession, '_create_session', + lambda s, url: cls(url)) + stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi', + fake_import) + + +class FakeSessionForVMTests(fake.SessionBase): + """ Stubs out a XenAPISession for VM tests """ + def __init__(self, uri): + super(FakeSessionForVMTests, self).__init__(uri) + + def network_get_all_records_where(self, _1, _2): + return self.xenapi.network.get_all_records() + + def host_call_plugin(self, _1, _2, _3, _4, _5): + return '' + + def VM_start(self, _1, ref, _2, _3): + vm = fake.get_record('VM', ref) + if vm['power_state'] != 'Halted': + raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted', + vm['power_state']]) + vm['power_state'] = 'Running' + + +class FakeSessionForVolumeTests(fake.SessionBase): + """ Stubs out a XenAPISession for Volume tests """ + def __init__(self, uri): + super(FakeSessionForVolumeTests, self).__init__(uri) + + def VBD_plug(self, _1, _2): + #FIXME(armando):make proper plug + pass + + def PBD_unplug(self, _1, _2): + #FIXME(armando):make proper unplug + pass + + def SR_forget(self, _1, _2): + #FIXME(armando):make proper forget + pass + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + #FIXME(armando):make proper introduce + valid_vdi = False + refs = fake.get_all('VDI') + for ref in refs: + rec = fake.get_record('VDI', ref) + if rec['uuid'] == uuid: + valid_vdi = True + if not valid_vdi: + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + +class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): + """ Stubs out a XenAPISession for Volume tests: it injects failures """ + def __init__(self, uri): + super(FakeSessionForVolumeFailedTests, self).__init__(uri) + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + # This is for testing failure + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + def VBD_plug(self, _1, _2): + # This is for testing failure + raise fake.Failure([['INVALID_VBD', 'session', self._session]]) diff --git a/nova/tests/xenapi_unittest.py b/nova/tests/xenapi_unittest.py index b9955a94..c2612a4c 100644 --- a/nova/tests/xenapi_unittest.py +++ b/nova/tests/xenapi_unittest.py @@ -1,21 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 -# # Copyright (c) 2010 Citrix Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -30,16 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +""" +Test suite for XenAPI +""" import stubout -import uuid - -from twisted.internet import defer -from twisted.internet import threads from nova import db from nova import context -from nova import exception from nova import flags from nova import test from nova import utils @@ -49,28 +31,15 @@ from nova.compute import power_state from nova.virt import xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils -from nova.virt.xenapi import vm_utils -from nova.virt.xenapi import volumeops -from boto.ec2.volume import Volume +from nova.tests.db import fakes +from nova.tests.xenapi import stubs FLAGS = flags.FLAGS -def stubout_session(stubs, cls): - def fake_import(self): - fake_module = 'nova.virt.xenapi.fake' - from_list = ['fake'] - return __import__(fake_module, globals(), locals(), from_list, -1) - - stubs.Set(xenapi_conn.XenAPISession, '_create_session', - lambda s, url: cls(url)) - stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi', - fake_import) - - class XenAPIVolumeTestCase(test.TrialTestCase): """ - Unit tests for VM operations + Unit tests for Volume operations """ def setUp(self): super(XenAPIVolumeTestCase, self).setUp() @@ -78,7 +47,17 @@ class XenAPIVolumeTestCase(test.TrialTestCase): FLAGS.target_host = '127.0.0.1' FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' + fakes.stub_out_db_instance_api(self.stubs) fake.reset() + self.values = {'name': 1, + 'project_id': 'fake', + 'user_id': 'fake', + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } def _create_volume(self, size='0'): """Create a volume object.""" @@ -94,7 +73,7 @@ class XenAPIVolumeTestCase(test.TrialTestCase): def test_create_iscsi_storage(self): """ This shows how to test helper classes' methods """ - stubout_session(self.stubs, FakeSessionForVolumeTests) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') helper = volume_utils.VolumeHelper helper.XenAPI = session.get_imported_xenapi() @@ -106,11 +85,13 @@ class XenAPIVolumeTestCase(test.TrialTestCase): info, label, description) + srs = fake.get_all('SR') + self.assertEqual(sr_ref, srs[0]) db.volume_destroy(context.get_admin_context(), vol['id']) def test_parse_volume_info_raise_exception(self): """ This shows how to test helper classes' methods """ - stubout_session(self.stubs, FakeSessionForVolumeTests) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') helper = volume_utils.VolumeHelper helper.XenAPI = session.get_imported_xenapi() @@ -119,6 +100,7 @@ class XenAPIVolumeTestCase(test.TrialTestCase): info = helper.parse_volume_info(vol['ec2_id'], '/dev/sd') def check(exc): + """ handler """ self.assertIsInstance(exc.value, volume_utils.StorageError) info.addErrback(check) @@ -126,16 +108,16 @@ class XenAPIVolumeTestCase(test.TrialTestCase): def test_attach_volume(self): """ This shows how to test Ops classes' methods """ - stubout_session(self.stubs, FakeSessionForVolumeTests) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) conn = xenapi_conn.get_connection(False) volume = self._create_volume() - instance = FakeInstance(1, 'fake', 'fake', 1, 2, 3, - 'm1.large', 'aa:bb:cc:dd:ee:ff') + instance = db.instance_create(self.values) fake.create_vm(instance.name, 'Running') result = conn.attach_volume(instance.name, volume['ec2_id'], '/dev/sdc') def check(_): + """ handler """ # check that the VM has a VBD attached to it # Get XenAPI reference for the VM vms = fake.get_all('VM') @@ -150,16 +132,17 @@ class XenAPIVolumeTestCase(test.TrialTestCase): def test_attach_volume_raise_exception(self): """ This shows how to test when exceptions are raised """ - stubout_session(self.stubs, FakeSessionForVolumeFailedTests) + stubs.stubout_session(self.stubs, + stubs.FakeSessionForVolumeFailedTests) conn = xenapi_conn.get_connection(False) volume = self._create_volume() - instance = FakeInstance(1, 'fake', 'fake', 1, 2, 3, - 'm1.large', 'aa:bb:cc:dd:ee:ff') + instance = db.instance_create(self.values) fake.create_vm(instance.name, 'Running') result = conn.attach_volume(instance.name, volume['ec2_id'], '/dev/sdc') def check(exc): + """ handler """ if exc: pass else: @@ -188,22 +171,32 @@ class XenAPIVMTestCase(test.TrialTestCase): FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' fake.reset() + fakes.stub_out_db_instance_api(self.stubs) fake.create_network('fake', FLAGS.flat_network_bridge) def test_list_instances_0(self): - stubout_session(self.stubs, FakeSessionForVMTests) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) conn = xenapi_conn.get_connection(False) instances = conn.list_instances() self.assertEquals(instances, []) def test_spawn(self): - stubout_session(self.stubs, FakeSessionForVMTests) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + values = {'name': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } conn = xenapi_conn.get_connection(False) - instance = FakeInstance(1, self.project.id, self.user.id, 1, 2, 3, - 'm1.large', 'aa:bb:cc:dd:ee:ff') + instance = db.instance_create(values) result = conn.spawn(instance) def check(_): + """ handler """ instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -241,77 +234,3 @@ class XenAPIVMTestCase(test.TrialTestCase): self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.stubs.UnsetAll() - - -class FakeInstance(): - def __init__(self, name, project_id, user_id, image_id, kernel_id, - ramdisk_id, instance_type, mac_address): - self.name = name - self.project_id = project_id - self.user_id = user_id - self.image_id = image_id - self.kernel_id = kernel_id - self.ramdisk_id = ramdisk_id - self.instance_type = instance_type - self.mac_address = mac_address - - -class FakeSessionForVMTests(fake.SessionBase): - def __init__(self, uri): - super(FakeSessionForVMTests, self).__init__(uri) - - def network_get_all_records_where(self, _1, _2): - return self.xenapi.network.get_all_records() - - def host_call_plugin(self, _1, _2, _3, _4, _5): - return '' - - def VM_start(self, _1, ref, _2, _3): - vm = fake.get_record('VM', ref) - if vm['power_state'] != 'Halted': - raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted', - vm['power_state']]) - vm['power_state'] = 'Running' - - -class FakeSessionForVolumeTests(fake.SessionBase): - def __init__(self, uri): - super(FakeSessionForVolumeTests, self).__init__(uri) - - def VBD_plug(self, _1, _2): - #FIXME(armando):make proper plug - pass - - def PBD_unplug(self, _1, _2): - #FIXME(armando):make proper unplug - pass - - def SR_forget(self, _1, _2): - #FIXME(armando):make proper forget - pass - - def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, - _6, _7, _8, _9, _10, _11): - #FIXME(armando):make proper introduce - valid_vdi = False - refs = fake.get_all('VDI') - for ref in refs: - rec = fake.get_record('VDI', ref) - if rec['uuid'] == uuid: - valid_vdi = True - if not valid_vdi: - raise fake.Failure([['INVALID_VDI', 'session', self._session]]) - - -class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): - def __init__(self, uri): - super(FakeSessionForVolumeFailedTests, self).__init__(uri) - - def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, - _6, _7, _8, _9, _10, _11): - # test failure - raise fake.Failure([['INVALID_VDI', 'session', self._session]]) - - def VBD_plug(self, _1, _2): - # test failure - raise fake.Failure([['INVALID_VBD', 'session', self._session]]) From e0b5318e54373ac1d53ad12e237ca0cb6795ed3f Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 16 Dec 2010 17:47:48 +0000 Subject: [PATCH 17/54] fake session clean-up --- nova/tests/xenapi/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/nova/tests/xenapi/__init__.py b/nova/tests/xenapi/__init__.py index a157d759..dcd81b74 100644 --- a/nova/tests/xenapi/__init__.py +++ b/nova/tests/xenapi/__init__.py @@ -1,21 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 -# # Copyright (c) 2010 Citrix Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may From 73e1126bc05e22688ff9ccfa62ebaa5a1c6412ea Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 16 Dec 2010 18:44:42 +0000 Subject: [PATCH 18/54] reviewed the FIXMEs, and spotted an uncaught exception in volume_utils...yay! --- nova/tests/virt_unittest.py | 2 -- nova/tests/xenapi/stubs.py | 24 +++++++++--------------- nova/tests/xenapi_unittest.py | 5 ++--- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index 52843b70..d49383fb 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -25,8 +25,6 @@ from nova import utils from nova.api.ec2 import cloud from nova.auth import manager from nova.virt import libvirt_conn -from nova.virt.xenapi import fake -from nova.virt.xenapi import volume_utils FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 52518938..11dd535d 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -58,21 +58,12 @@ class FakeSessionForVolumeTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForVolumeTests, self).__init__(uri) - def VBD_plug(self, _1, _2): - #FIXME(armando):make proper plug - pass - - def PBD_unplug(self, _1, _2): - #FIXME(armando):make proper unplug - pass - - def SR_forget(self, _1, _2): - #FIXME(armando):make proper forget - pass + def VBD_plug(self, _1, ref): + rec = fake.get_record('VBD', ref) + rec['currently-attached'] = True def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11): - #FIXME(armando):make proper introduce valid_vdi = False refs = fake.get_all('VDI') for ref in refs: @@ -93,6 +84,9 @@ class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): # This is for testing failure raise fake.Failure([['INVALID_VDI', 'session', self._session]]) - def VBD_plug(self, _1, _2): - # This is for testing failure - raise fake.Failure([['INVALID_VBD', 'session', self._session]]) + def PBD_unplug(self, _1, ref): + rec = fake.get_record('PBD', ref) + rec['currently-attached'] = False + + def SR_forget(self, _1, ref): + pass diff --git a/nova/tests/xenapi_unittest.py b/nova/tests/xenapi_unittest.py index c2612a4c..839d6aa4 100644 --- a/nova/tests/xenapi_unittest.py +++ b/nova/tests/xenapi_unittest.py @@ -141,14 +141,13 @@ class XenAPIVolumeTestCase(test.TrialTestCase): result = conn.attach_volume(instance.name, volume['ec2_id'], '/dev/sdc') - def check(exc): + def check_exception(exc): """ handler """ if exc: pass else: self.fail('Oops, no exception has been raised!') - - result.addErrback(check) + result.addErrback(check_exception) return result def tearDown(self): From 524a16dda6561cf70a058c4db5cda706c76cddae Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 16 Dec 2010 17:05:54 -0800 Subject: [PATCH 19/54] First pass at converting run_tests.py to nosetests. The network and objctstore tests don't yet work. Also, we need to manually remove the sqlite file between runs. --- nova/tests/api_integration.py | 54 ----- nova/tests/quota_unittest.py | 153 ------------ nova/tests/service_unittest.py | 227 ------------------ .../{access_unittest.py => test_access.py} | 0 nova/tests/{api_unittest.py => test_api.py} | 0 nova/tests/{auth_unittest.py => test_auth.py} | 0 .../{cloud_unittest.py => test_cloud.py} | 0 .../{compute_unittest.py => test_compute.py} | 0 .../{flags_unittest.py => test_flags.py} | 0 nova/tests/{misc_unittest.py => test_misc.py} | 8 +- nova/tests/{rpc_unittest.py => test_rpc.py} | 0 ...cheduler_unittest.py => test_scheduler.py} | 2 +- .../{twistd_unittest.py => test_twistd.py} | 0 nova/tests/{virt_unittest.py => test_virt.py} | 0 .../{volume_unittest.py => test_volume.py} | 0 run_tests.py | 125 ---------- 16 files changed, 5 insertions(+), 564 deletions(-) delete mode 100644 nova/tests/api_integration.py delete mode 100644 nova/tests/quota_unittest.py delete mode 100644 nova/tests/service_unittest.py rename nova/tests/{access_unittest.py => test_access.py} (100%) rename nova/tests/{api_unittest.py => test_api.py} (100%) rename nova/tests/{auth_unittest.py => test_auth.py} (100%) rename nova/tests/{cloud_unittest.py => test_cloud.py} (100%) rename nova/tests/{compute_unittest.py => test_compute.py} (100%) rename nova/tests/{flags_unittest.py => test_flags.py} (100%) rename nova/tests/{misc_unittest.py => test_misc.py} (89%) rename nova/tests/{rpc_unittest.py => test_rpc.py} (100%) rename nova/tests/{scheduler_unittest.py => test_scheduler.py} (99%) rename nova/tests/{twistd_unittest.py => test_twistd.py} (100%) rename nova/tests/{virt_unittest.py => test_virt.py} (100%) rename nova/tests/{volume_unittest.py => test_volume.py} (100%) delete mode 100644 run_tests.py diff --git a/nova/tests/api_integration.py b/nova/tests/api_integration.py deleted file mode 100644 index 54403c65..00000000 --- a/nova/tests/api_integration.py +++ /dev/null @@ -1,54 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import boto -from boto.ec2.regioninfo import RegionInfo -import unittest - - -ACCESS_KEY = 'fake' -SECRET_KEY = 'fake' -CLC_IP = '127.0.0.1' -CLC_PORT = 8773 -REGION = 'test' - - -def get_connection(): - return boto.connect_ec2( - aws_access_key_id=ACCESS_KEY, - aws_secret_access_key=SECRET_KEY, - is_secure=False, - region=RegionInfo(None, REGION, CLC_IP), - port=CLC_PORT, - path='/services/Cloud', - debug=99) - - -class APIIntegrationTests(unittest.TestCase): - def test_001_get_all_images(self): - conn = get_connection() - res = conn.get_all_images() - - -if __name__ == '__main__': - unittest.main() - -#print conn.get_all_key_pairs() -#print conn.create_key_pair -#print conn.create_security_group('name', 'description') diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py deleted file mode 100644 index 8cf2a5e5..00000000 --- a/nova/tests/quota_unittest.py +++ /dev/null @@ -1,153 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -from nova import context -from nova import db -from nova import exception -from nova import flags -from nova import quota -from nova import test -from nova import utils -from nova.auth import manager -from nova.api.ec2 import cloud - - -FLAGS = flags.FLAGS - - -class QuotaTestCase(test.TestCase): - def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) - super(QuotaTestCase, self).setUp() - self.flags(connection_type='fake', - quota_instances=2, - quota_cores=4, - quota_volumes=2, - quota_gigabytes=20, - quota_floating_ips=1) - - self.cloud = cloud.CloudController() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('admin', 'admin', 'admin', True) - self.project = self.manager.create_project('admin', 'admin', 'admin') - self.network = utils.import_object(FLAGS.network_manager) - self.context = context.RequestContext(project=self.project, - user=self.user) - - def tearDown(self): - manager.AuthManager().delete_project(self.project) - manager.AuthManager().delete_user(self.user) - super(QuotaTestCase, self).tearDown() - - def _create_instance(self, cores=2): - """Create a test instance""" - inst = {} - inst['image_id'] = 'ami-test' - inst['reservation_id'] = 'r-fakeres' - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id - inst['instance_type'] = 'm1.large' - inst['vcpus'] = cores - inst['mac_address'] = utils.generate_mac() - return db.instance_create(self.context, inst)['id'] - - def _create_volume(self, size=10): - """Create a test volume""" - vol = {} - vol['user_id'] = self.user.id - vol['project_id'] = self.project.id - vol['size'] = size - return db.volume_create(self.context, vol)['id'] - - def test_quota_overrides(self): - """Make sure overriding a projects quotas works""" - num_instances = quota.allowed_instances(self.context, 100, 'm1.small') - self.assertEqual(num_instances, 2) - db.quota_create(self.context, {'project_id': self.project.id, - 'instances': 10}) - num_instances = quota.allowed_instances(self.context, 100, 'm1.small') - self.assertEqual(num_instances, 4) - db.quota_update(self.context, self.project.id, {'cores': 100}) - num_instances = quota.allowed_instances(self.context, 100, 'm1.small') - self.assertEqual(num_instances, 10) - db.quota_destroy(self.context, self.project.id) - - def test_too_many_instances(self): - instance_ids = [] - for i in range(FLAGS.quota_instances): - instance_id = self._create_instance() - instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, - self.context, - min_count=1, - max_count=1, - instance_type='m1.small', - image_id='fake') - for instance_id in instance_ids: - db.instance_destroy(self.context, instance_id) - - def test_too_many_cores(self): - instance_ids = [] - instance_id = self._create_instance(cores=4) - instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, - self.context, - min_count=1, - max_count=1, - instance_type='m1.small', - image_id='fake') - for instance_id in instance_ids: - db.instance_destroy(self.context, instance_id) - - def test_too_many_volumes(self): - volume_ids = [] - for i in range(FLAGS.quota_volumes): - volume_id = self._create_volume() - volume_ids.append(volume_id) - self.assertRaises(quota.QuotaError, self.cloud.create_volume, - self.context, - size=10) - for volume_id in volume_ids: - db.volume_destroy(self.context, volume_id) - - def test_too_many_gigabytes(self): - volume_ids = [] - volume_id = self._create_volume(size=20) - volume_ids.append(volume_id) - self.assertRaises(quota.QuotaError, - self.cloud.create_volume, - self.context, - size=10) - for volume_id in volume_ids: - db.volume_destroy(self.context, volume_id) - - def test_too_many_addresses(self): - address = '192.168.0.100' - db.floating_ip_create(context.get_admin_context(), - {'address': address, 'host': FLAGS.host}) - float_addr = self.network.allocate_floating_ip(self.context, - self.project.id) - # NOTE(vish): This assert never fails. When cloud attempts to - # make an rpc.call, the test just finishes with OK. It - # appears to be something in the magic inline callbacks - # that is breaking. - self.assertRaises(quota.QuotaError, self.cloud.allocate_address, - self.context) - db.floating_ip_destroy(context.get_admin_context(), address) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py deleted file mode 100644 index 47c092f8..00000000 --- a/nova/tests/service_unittest.py +++ /dev/null @@ -1,227 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Unit Tests for remote procedure calls using queue -""" - -import mox - -from nova import exception -from nova import flags -from nova import rpc -from nova import test -from nova import service -from nova import manager - -FLAGS = flags.FLAGS -flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager", - "Manager for testing") - - -class FakeManager(manager.Manager): - """Fake manager for tests""" - def test_method(self): - return 'manager' - - -class ExtendedService(service.Service): - def test_method(self): - return 'service' - - -class ServiceManagerTestCase(test.TestCase): - """Test cases for Services""" - - def test_attribute_error_for_no_manager(self): - serv = service.Service('test', - 'test', - 'test', - 'nova.tests.service_unittest.FakeManager') - self.assertRaises(AttributeError, getattr, serv, 'test_method') - - def test_message_gets_to_manager(self): - serv = service.Service('test', - 'test', - 'test', - 'nova.tests.service_unittest.FakeManager') - serv.start() - self.assertEqual(serv.test_method(), 'manager') - - def test_override_manager_method(self): - serv = ExtendedService('test', - 'test', - 'test', - 'nova.tests.service_unittest.FakeManager') - serv.start() - self.assertEqual(serv.test_method(), 'service') - - -class ServiceTestCase(test.TestCase): - """Test cases for Services""" - - def setUp(self): - super(ServiceTestCase, self).setUp() - self.mox.StubOutWithMock(service, 'db') - - def test_create(self): - host = 'foo' - binary = 'nova-fake' - topic = 'fake' - - # NOTE(vish): Create was moved out of mox replay to make sure that - # the looping calls are created in StartService. - app = service.Service.create(host=host, binary=binary) - - self.mox.StubOutWithMock(rpc, - 'AdapterConsumer', - use_mock_anything=True) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic=topic, - proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) - - rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='%s.%s' % (topic, host), - proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) - - rpc.AdapterConsumer.attach_to_eventlet() - rpc.AdapterConsumer.attach_to_eventlet() - - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - self.mox.ReplayAll() - - app.start() - app.stop() - self.assert_(app) - - # We're testing sort of weird behavior in how report_state decides - # whether it is disconnected, it looks for a variable on itself called - # 'model_disconnected' and report_state doesn't really do much so this - # these are mostly just for coverage - def test_report_state_no_service(self): - host = 'foo' - binary = 'bar' - topic = 'test' - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - service.db.service_get(mox.IgnoreArg(), - service_ref['id']).AndReturn(service_ref) - service.db.service_update(mox.IgnoreArg(), service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - serv = service.Service(host, - binary, - topic, - 'nova.tests.service_unittest.FakeManager') - serv.start() - serv.report_state() - - def test_report_state_newly_disconnected(self): - host = 'foo' - binary = 'bar' - topic = 'test' - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - service.db.service_get(mox.IgnoreArg(), - mox.IgnoreArg()).AndRaise(Exception()) - - self.mox.ReplayAll() - serv = service.Service(host, - binary, - topic, - 'nova.tests.service_unittest.FakeManager') - serv.start() - serv.report_state() - self.assert_(serv.model_disconnected) - - def test_report_state_newly_connected(self): - host = 'foo' - binary = 'bar' - topic = 'test' - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - service.db.service_get(mox.IgnoreArg(), - service_ref['id']).AndReturn(service_ref) - service.db.service_update(mox.IgnoreArg(), service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - serv = service.Service(host, - binary, - topic, - 'nova.tests.service_unittest.FakeManager') - serv.start() - serv.model_disconnected = True - serv.report_state() - - self.assert_(not serv.model_disconnected) diff --git a/nova/tests/access_unittest.py b/nova/tests/test_access.py similarity index 100% rename from nova/tests/access_unittest.py rename to nova/tests/test_access.py diff --git a/nova/tests/api_unittest.py b/nova/tests/test_api.py similarity index 100% rename from nova/tests/api_unittest.py rename to nova/tests/test_api.py diff --git a/nova/tests/auth_unittest.py b/nova/tests/test_auth.py similarity index 100% rename from nova/tests/auth_unittest.py rename to nova/tests/test_auth.py diff --git a/nova/tests/cloud_unittest.py b/nova/tests/test_cloud.py similarity index 100% rename from nova/tests/cloud_unittest.py rename to nova/tests/test_cloud.py diff --git a/nova/tests/compute_unittest.py b/nova/tests/test_compute.py similarity index 100% rename from nova/tests/compute_unittest.py rename to nova/tests/test_compute.py diff --git a/nova/tests/flags_unittest.py b/nova/tests/test_flags.py similarity index 100% rename from nova/tests/flags_unittest.py rename to nova/tests/test_flags.py diff --git a/nova/tests/misc_unittest.py b/nova/tests/test_misc.py similarity index 89% rename from nova/tests/misc_unittest.py rename to nova/tests/test_misc.py index 3d947427..33c1777d 100644 --- a/nova/tests/misc_unittest.py +++ b/nova/tests/test_misc.py @@ -22,13 +22,13 @@ from nova.utils import parse_mailmap, str_dict_replace class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): - if os.path.exists('../.bzr'): + if os.path.exists('.bzr'): contributors = set() - mailmap = parse_mailmap('../.mailmap') + mailmap = parse_mailmap('.mailmap') import bzrlib.workingtree - tree = bzrlib.workingtree.WorkingTree.open('..') + tree = bzrlib.workingtree.WorkingTree.open('.') tree.lock_read() try: parents = tree.get_parent_ids() @@ -42,7 +42,7 @@ class ProjectTestCase(test.TestCase): email = author.split(' ')[-1] contributors.add(str_dict_replace(email, mailmap)) - authors_file = open('../Authors', 'r').read() + authors_file = open('Authors', 'r').read() missing = set() for contributor in contributors: diff --git a/nova/tests/rpc_unittest.py b/nova/tests/test_rpc.py similarity index 100% rename from nova/tests/rpc_unittest.py rename to nova/tests/test_rpc.py diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/test_scheduler.py similarity index 99% rename from nova/tests/scheduler_unittest.py rename to nova/tests/test_scheduler.py index d1756b8f..c56f6969 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/test_scheduler.py @@ -48,7 +48,7 @@ class SchedulerTestCase(test.TestCase): """Test case for scheduler""" def setUp(self): super(SchedulerTestCase, self).setUp() - self.flags(scheduler_driver='nova.tests.scheduler_unittest.TestDriver') + self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def test_fallback(self): scheduler = manager.SchedulerManager() diff --git a/nova/tests/twistd_unittest.py b/nova/tests/test_twistd.py similarity index 100% rename from nova/tests/twistd_unittest.py rename to nova/tests/test_twistd.py diff --git a/nova/tests/virt_unittest.py b/nova/tests/test_virt.py similarity index 100% rename from nova/tests/virt_unittest.py rename to nova/tests/test_virt.py diff --git a/nova/tests/volume_unittest.py b/nova/tests/test_volume.py similarity index 100% rename from nova/tests/volume_unittest.py rename to nova/tests/test_volume.py diff --git a/run_tests.py b/run_tests.py deleted file mode 100644 index 6a4b7f1a..00000000 --- a/run_tests.py +++ /dev/null @@ -1,125 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -This is our basic test running framework based on Twisted's Trial. - -Usage Examples: - - # to run all the tests - python run_tests.py - - # to run a specific test suite imported here - python run_tests.py NodeConnectionTestCase - - # to run a specific test imported here - python run_tests.py NodeConnectionTestCase.test_reboot - - # to run some test suites elsewhere - python run_tests.py nova.tests.node_unittest - python run_tests.py nova.tests.node_unittest.NodeConnectionTestCase - -Due to our use of multiprocessing it we frequently get some ignorable -'Interrupted system call' exceptions after test completion. - -""" - -import eventlet -eventlet.monkey_patch() - -import __main__ -import gettext -import os -import sys - -gettext.install('nova', unicode=1) - -from twisted.scripts import trial as trial_script - -from nova import flags -from nova import twistd - -from nova.tests.access_unittest import * -from nova.tests.api_unittest import * -from nova.tests.auth_unittest import * -from nova.tests.cloud_unittest import * -from nova.tests.compute_unittest import * -from nova.tests.flags_unittest import * -from nova.tests.misc_unittest import * -from nova.tests.network_unittest import * -#from nova.tests.objectstore_unittest import * -from nova.tests.quota_unittest import * -from nova.tests.rpc_unittest import * -from nova.tests.scheduler_unittest import * -from nova.tests.service_unittest import * -from nova.tests.twistd_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * - - -FLAGS = flags.FLAGS -flags.DEFINE_bool('flush_db', True, - 'Flush the database before running fake tests') -flags.DEFINE_string('tests_stderr', 'run_tests.err.log', - 'Path to where to pipe STDERR during test runs.' - ' Default = "run_tests.err.log"') - - -if __name__ == '__main__': - OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) - config = OptionsClass() - argv = config.parseOptions() - - FLAGS.verbose = True - - # TODO(termie): these should make a call instead of doing work on import - if FLAGS.fake_tests: - from nova.tests.fake_flags import * - else: - from nova.tests.real_flags import * - - # Establish redirect for STDERR - sys.stderr.flush() - err = open(FLAGS.tests_stderr, 'w+', 0) - os.dup2(err.fileno(), sys.stderr.fileno()) - - if len(argv) == 1 and len(config['tests']) == 0: - # If no tests were specified run the ones imported in this file - # NOTE(termie): "tests" is not a flag, just some Trial related stuff - config['tests'].update(['__main__']) - elif len(config['tests']): - # If we specified tests check first whether they are in __main__ - for arg in config['tests']: - key = arg.split('.')[0] - if hasattr(__main__, key): - config['tests'].remove(arg) - config['tests'].add('__main__.%s' % arg) - - trial_script._initialDebugSetup(config) - trialRunner = trial_script._makeRunner(config) - suite = trial_script._getSuite(config) - if config['until-failure']: - test_result = trialRunner.runUntilFailure(suite) - else: - test_result = trialRunner.run(suite) - if config.tracer: - sys.settrace(None) - results = config.tracer.results() - results.write_results(show_missing=1, summary=False, - coverdir=config.coverdir) - sys.exit(not test_result.wasSuccessful()) From 430e97b84862596fe79a097e4b092ca9b54d3385 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 16 Dec 2010 17:29:26 -0800 Subject: [PATCH 20/54] Fixed network test (thanks Vish!) and fixed run_tests.sh. --- bin/nova-dhcpbridge | 1 - nova/tests/{network_unittest.py => test_network.py} | 0 2 files changed, 1 deletion(-) rename nova/tests/{network_unittest.py => test_network.py} (100%) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 81b9b6dd..828aba3d 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -110,7 +110,6 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - '_trial_temp', 'nova.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] diff --git a/nova/tests/network_unittest.py b/nova/tests/test_network.py similarity index 100% rename from nova/tests/network_unittest.py rename to nova/tests/test_network.py From 9c8255b2cc824c298cb6bdf15bbf4574691309fd Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Sat, 18 Dec 2010 00:50:49 +0000 Subject: [PATCH 21/54] fixed unittests and further clean-up post-eventlet merge --- nova/tests/xenapi_unittest.py | 48 ++++++++++++----------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/nova/tests/xenapi_unittest.py b/nova/tests/xenapi_unittest.py index 839d6aa4..74401ce5 100644 --- a/nova/tests/xenapi_unittest.py +++ b/nova/tests/xenapi_unittest.py @@ -78,13 +78,10 @@ class XenAPIVolumeTestCase(test.TrialTestCase): helper = volume_utils.VolumeHelper helper.XenAPI = session.get_imported_xenapi() vol = self._create_volume() - info = yield helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') + info = helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') label = 'SR-%s' % vol['ec2_id'] description = 'Test-SR' - sr_ref = helper.create_iscsi_storage_blocking(session, - info, - label, - description) + sr_ref = helper.create_iscsi_storage(session, info, label, description) srs = fake.get_all('SR') self.assertEqual(sr_ref, srs[0]) db.volume_destroy(context.get_admin_context(), vol['id']) @@ -97,13 +94,10 @@ class XenAPIVolumeTestCase(test.TrialTestCase): helper.XenAPI = session.get_imported_xenapi() vol = self._create_volume() # oops, wrong mount point! - info = helper.parse_volume_info(vol['ec2_id'], '/dev/sd') - - def check(exc): - """ handler """ - self.assertIsInstance(exc.value, volume_utils.StorageError) - - info.addErrback(check) + self.assertRaises(volume_utils.StorageError, + helper.parse_volume_info, + vol['ec2_id'], + '/dev/sd') db.volume_destroy(context.get_admin_context(), vol['id']) def test_attach_volume(self): @@ -116,8 +110,7 @@ class XenAPIVolumeTestCase(test.TrialTestCase): result = conn.attach_volume(instance.name, volume['ec2_id'], '/dev/sdc') - def check(_): - """ handler """ + def check(): # check that the VM has a VBD attached to it # Get XenAPI reference for the VM vms = fake.get_all('VM') @@ -127,8 +120,7 @@ class XenAPIVolumeTestCase(test.TrialTestCase): vm_ref = vbd['VM'] self.assertEqual(vm_ref, vms[0]) - result.addCallback(check) - return result + check() def test_attach_volume_raise_exception(self): """ This shows how to test when exceptions are raised """ @@ -138,17 +130,11 @@ class XenAPIVolumeTestCase(test.TrialTestCase): volume = self._create_volume() instance = db.instance_create(self.values) fake.create_vm(instance.name, 'Running') - result = conn.attach_volume(instance.name, volume['ec2_id'], - '/dev/sdc') - - def check_exception(exc): - """ handler """ - if exc: - pass - else: - self.fail('Oops, no exception has been raised!') - result.addErrback(check_exception) - return result + self.assertRaises(Exception, + conn.attach_volume, + instance.name, + volume['ec2_id'], + '/dev/sdc') def tearDown(self): super(XenAPIVolumeTestCase, self).tearDown() @@ -192,10 +178,9 @@ class XenAPIVMTestCase(test.TrialTestCase): } conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) - result = conn.spawn(instance) + conn.spawn(instance) - def check(_): - """ handler """ + def check(): instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -225,8 +210,7 @@ class XenAPIVMTestCase(test.TrialTestCase): # Check that the VM is running according to XenAPI. self.assertEquals(vm['power_state'], 'Running') - result.addCallback(check) - return result + check() def tearDown(self): super(XenAPIVMTestCase, self).tearDown() From 781518b6ca3090549a3c5e7f60ed150f3cc5fe98 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Fri, 17 Dec 2010 19:17:39 -0600 Subject: [PATCH 22/54] Cleaned up TODOs, using flags now --- nova/flags.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 87444565..3318af05 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -212,6 +212,9 @@ DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') +DEFINE_integer('glance_port', 9292, 'glance port') +DEFINE_string('glance_host', '127.0.0.1', 'glance host') +DEFINE_string('glance_storage_location', 'swift://username:api_key@auth.api.rackspacecloud.com/v1.0/cloudservers', 'glance storage location') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') From b01b8a802c28f7f48c04aa05b7a0a7e55f044e3f Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Mon, 20 Dec 2010 15:41:43 +0000 Subject: [PATCH 23/54] Fixed pep8 errors --- nova/tests/virt_unittest.py | 43 +++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index 9bbba4ba..e5d5f00c 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -129,44 +129,41 @@ class LibvirtConnTestCase(test.TestCase): check_list.append(check) else: if expect_kernel: - check = (lambda t: t.find('./os/kernel').text.split('/' - )[1], 'kernel') + check = (lambda t: t.find('./os/kernel').text.split('/') + [1], 'kernel') else: check = (lambda t: t.find('./os/kernel'), None) check_list.append(check) if expect_ramdisk: - check = (lambda t: t.find('./os/initrd').text.split('/' - )[1], 'ramdisk') + check = (lambda t: t.find('./os/initrd').text.split('/') + [1], 'ramdisk') else: check = (lambda t: t.find('./os/initrd'), None) check_list.append(check) common_checks = [ (lambda t: t.find('.').tag, 'domain'), - (lambda t: t.find('./devices/interface/filterref/parameter' - ).get('name'), 'IP'), - (lambda t: t.find('./devices/interface/filterref/parameter' - ).get('value'), '10.11.12.13'), - (lambda t: t.findall('./devices/interface/filterref/parameter' - )[1].get('name'), 'DHCPSERVER'), - (lambda t: t.findall('./devices/interface/filterref/parameter' - )[1].get('value'), '10.0.0.1'), - (lambda t: t.find('./devices/serial/source').get('path' - ).split('/')[1], 'console.log'), + (lambda t: t.find('./devices/interface/filterref/parameter') + .get('name'), 'IP'), + (lambda t: t.find('./devices/interface/filterref/parameter') + .get('value'), '10.11.12.13'), + (lambda t: t.findall('./devices/interface/filterref/parameter') + [1].get('name'), 'DHCPSERVER'), + (lambda t: t.findall('./devices/interface/filterref/parameter') + [1].get('value'), '10.0.0.1'), + (lambda t: t.find('./devices/serial/source').get('path') + .split('/')[1], 'console.log'), (lambda t: t.find('./memory').text, '2097152')] if rescue: - common_checks += [(lambda t: t.findall('./devices/disk/source' - )[0].get('file').split('/')[1], - 'rescue-disk'), - (lambda t: t.findall('./devices/disk/source' - )[1].get('file').split('/')[1], - 'disk')] + common_checks += [(lambda t: t.findall('./devices/disk/source') + [0].get('file').split('/')[1], 'rescue-disk'), + (lambda t: t.findall('./devices/disk/source') + [1].get('file').split('/')[1], 'disk')] else: - common_checks += [(lambda t: t.findall('./devices/disk/source' - )[0].get('file').split('/')[1], - 'disk')] + common_checks += [(lambda t: t.findall('./devices/disk/source') + [0].get('file').split('/')[1], 'disk')] for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type From 714f77a3153ffacc015904a63047a486f2671bd4 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 20 Dec 2010 22:55:11 +0000 Subject: [PATCH 24/54] added suspend and resume --- nova/tests/compute_unittest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 187ca31d..111a43cd 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -135,6 +135,14 @@ class ComputeTestCase(test.TestCase): self.compute.unpause_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) + def test_suspend(self): + """ensure instance can be suspended""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.suspend_instance(self.context, instance_id) + self.compute.resume_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + def test_reboot(self): """Ensure instance can be rebooted""" instance_id = self._create_instance() From 660b974b92a09fe56a486d33809b2232ce2cae4e Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 21 Dec 2010 01:41:28 -0500 Subject: [PATCH 25/54] Use paste.deploy for running the api server. --- bin/nova-api-paste | 103 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 bin/nova-api-paste diff --git a/bin/nova-api-paste b/bin/nova-api-paste new file mode 100755 index 00000000..dcb76522 --- /dev/null +++ b/bin/nova-api-paste @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# pylint: disable-msg=C0103 +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starter script for Nova API.""" + +import gettext +import logging +import os +import sys + +from paste import deploy + +from nova import flags +from nova import wsgi + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +gettext.install('nova', unicode=1) + +LOG = logging.getLogger('nova.api') +LOG.setLevel(logging.DEBUG) +LOG.addHandler(logging.StreamHandler()) + +FLAGS = flags.FLAGS + +API_ENDPOINTS = ['ec2', 'openstack'] + +def load_configuration(paste_config): + """Load the paste configuration from the config file and return it.""" + config = None + # Try each known name to get the global DEFAULTS, which will give ports + for name in API_ENDPOINTS: + try: + config = deploy.appconfig("config:%s" % paste_config, name=name) + except LookupError: + pass + if config: + verbose = config.get('verbose', None) + if verbose: + FLAGS.verbose = int(verbose) == 1 + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + return config + LOG.debug("Paste config at %s has no secion for known apis", paste_config) + print "Paste config at %s has no secion for any known apis" % paste_config + os.exit(1) + +def launch_api(paste_config_file, section, server, port, host): + """Launch an api server from the specified port and IP.""" + LOG.debug("Launching api %s on %s:%s", section, host, port) + app = deploy.loadapp('config:%s' % paste_config_file, name=section) + server.start(app, int(port), host) + +def run_app(paste_config_file): + LOG.debug("Using paste.deploy config at: %s", configfile) + config = load_configuration(paste_config_file) + LOG.debug("Configuration: %r", config) + server = wsgi.Server() + ip = config.get('host', None) + for api in API_ENDPOINTS: + port = config.get("%s_port" % api, None) + if not port: + continue + host = config.get("%s_host" % api, None) or ip or '0.0.0.0' + launch_api(configfile, api, server, port, host) + LOG.debug("All api servers launched, now waiting") + server.wait() + +if __name__ == '__main__': + FLAGS(sys.argv) + configfiles = ['/etc/nova/nova-api.conf'] + if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + configfiles.insert(0, + os.path.join(possible_topdir, 'etc', 'nova-api.conf')) + for configfile in configfiles: + if os.path.exists(configfile): + run_app(configfile) + break + else: + LOG.debug("Skipping missing configuration: %s", configfile) From 5033154e00a66853c6359baa7150e67d39738cba Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 21 Dec 2010 19:20:28 -0500 Subject: [PATCH 26/54] Burnin support by specifying a specific host via availability_zone for running instances and volumes on. --- bin/nova-manage | 50 +++++++++++++- nova/scheduler/simple.py | 26 +++++++ nova/tests/scheduler_unittest.py | 112 +++++++++++++++++++++++++++++-- 3 files changed, 183 insertions(+), 5 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 0c1b621e..34bdd3df 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -53,6 +53,7 @@ CLI interface for nova management. """ +import datetime import gettext import logging import os @@ -432,6 +433,52 @@ class NetworkCommands(object): int(network_size), int(vlan_start), int(vpn_start)) + +class ServiceCommands(object): + """Enable and disable running services""" + + def list(self, host=None, service=None): + """Show a list of all running services. Filter by host & service name. + args: [host] [service]""" + ctxt = context.get_admin_context() + now = datetime.datetime.utcnow() + services = db.service_get_all(ctxt) + if host: + services = [s for s in services if s['host'] == host] + if service: + services = [s for s in services if s['binary'] == service] + for svc in services: + delta = now - (svc['updated_at'] or svc['created_at']) + alive = (delta.seconds <= 15) + art = (alive and ":-)") or "XXX" + active = 'enabled' + if svc['disabled']: + active = 'disabled' + print "%-10s %-10s %-8s %s %s" % (svc['host'], svc['binary'], + active, art, + svc['updated_at']) + + def enable(self, host, service): + """Enable scheduling for a service + args: host service""" + ctxt = context.get_admin_context() + svc = db.service_get_by_args(ctxt, host, service) + if not svc: + print "Unable to find service" + return + db.service_update(ctxt, svc['id'], {'disabled': False}) + + def disable(self, host, service): + """Disable scheduling for a service + args: host service""" + ctxt = context.get_admin_context() + svc = db.service_get_by_args(ctxt, host, service) + if not svc: + print "Unable to find service" + return + db.service_update(ctxt, svc['id'], {'disabled': True}) + + CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), @@ -439,7 +486,8 @@ CATEGORIES = [ ('shell', ShellCommands), ('vpn', VpnCommands), ('floating', FloatingIpCommands), - ('network', NetworkCommands)] + ('network', NetworkCommands), + ('service', ServiceCommands)] def lazy_match(name, key_value_tuples): diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 7f509365..9e85ba95 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -43,6 +43,19 @@ class SimpleScheduler(chance.ChanceScheduler): def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" instance_ref = db.instance_get(context, instance_id) + if instance_ref['availability_zone'] and context.is_admin: + zone, _x, host = instance_ref['availability_zone'].partition(':') + service = db.service_get_by_args(context.elevated(), host, + 'nova-compute') + if not self.service_is_up(service): + raise driver.WillNotSchedule("Host %s is not alive" % host) + + # TODO(vish): this probably belongs in the manager, if we + # can generalize this somehow + now = datetime.datetime.utcnow() + db.instance_update(context, instance_id, {'host': host, + 'scheduled_at': now}) + return host results = db.service_get_all_compute_sorted(context) for result in results: (service, instance_cores) = result @@ -62,6 +75,19 @@ class SimpleScheduler(chance.ChanceScheduler): def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" volume_ref = db.volume_get(context, volume_id) + if (':' in volume_ref['availability_zone']) and context.is_admin: + zone, _x, host = volume_ref['availability_zone'].partition(':') + service = db.service_get_by_args(context.elevated(), host, + 'nova-volume') + if not self.service_is_up(service): + raise driver.WillNotSchedule("Host %s not available" % host) + + # TODO(vish): this probably belongs in the manager, if we + # can generalize this somehow + now = datetime.datetime.utcnow() + db.volume_update(context, volume_id, {'host': host, + 'scheduled_at': now}) + return host results = db.service_get_all_volume_sorted(context) for result in results: (service, volume_gigabytes) = result diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index d1756b8f..92262cc7 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -19,6 +19,8 @@ Tests For Scheduler """ +import datetime + from nova import context from nova import db from nova import flags @@ -93,7 +95,7 @@ class SimpleDriverTestCase(test.TestCase): self.manager.delete_user(self.user) self.manager.delete_project(self.project) - def _create_instance(self): + def _create_instance(self, **kwargs): """Create a test instance""" inst = {} inst['image_id'] = 'ami-test' @@ -104,6 +106,7 @@ class SimpleDriverTestCase(test.TestCase): inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 inst['vcpus'] = 1 + inst['availability_zone'] = kwargs.get('availability_zone', None) return db.instance_create(self.context, inst)['id'] def _create_volume(self): @@ -112,10 +115,11 @@ class SimpleDriverTestCase(test.TestCase): vol['image_id'] = 'ami-test' vol['reservation_id'] = 'r-fakeres' vol['size'] = 1 + vol['availability_zone'] = 'test' return db.volume_create(self.context, vol)['id'] - def test_hosts_are_up(self): - """Ensures driver can find the hosts that are up""" + def test_doesnt_report_disabled_hosts_as_up(self): + """Ensures driver doesn't find hosts before they are enabled""" # NOTE(vish): constructing service without create method # because we are going to use it without queue compute1 = service.Service('host1', @@ -129,7 +133,30 @@ class SimpleDriverTestCase(test.TestCase): FLAGS.compute_manager) compute2.start() hosts = self.scheduler.driver.hosts_up(self.context, 'compute') - self.assertEqual(len(hosts), 2) + self.assertEqual(0, len(hosts)) + compute1.kill() + compute2.kill() + + def test_reports_enabled_hosts_as_up(self): + """Ensures driver can find the hosts that are up""" + # NOTE(vish): constructing service without create method + # because we are going to use it without queue + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) + hosts = self.scheduler.driver.hosts_up(self.context, 'compute') + self.assertEqual(2, len(hosts)) compute1.kill() compute2.kill() @@ -145,6 +172,10 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -156,6 +187,67 @@ class SimpleDriverTestCase(test.TestCase): compute1.kill() compute2.kill() + def test_specific_host_gets_instance(self): + """Ensures if you set availability_zone it launches on that zone""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) + instance_id1 = self._create_instance() + compute1.run_instance(self.context, instance_id1) + instance_id2 = self._create_instance(availability_zone='nova:host1') + host = self.scheduler.driver.schedule_run_instance(self.context, + instance_id2) + self.assertEqual('host1', host) + compute1.terminate_instance(self.context, instance_id1) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + compute2.kill() + + def test_wont_sechedule_if_specified_host_is_down(self): + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) + past = now - delta + db.service_update(self.context, s1['id'], {'disabled': False, + 'updated_at': past}) + instance_id2 = self._create_instance(availability_zone='nova:host1') + self.assertRaises(driver.WillNotSchedule, + self.scheduler.driver.schedule_run_instance, + self.context, + instance_id2) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + + def test_will_schedule_on_disabled_host_if_specified(self): + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + db.service_get_by_args(self.context, 'host1', 'nova-compute') + instance_id2 = self._create_instance(availability_zone='nova:host1') + host = self.scheduler.driver.schedule_run_instance(self.context, + instance_id2) + self.assertEqual('host1', host) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + def test_too_many_cores(self): """Ensures we don't go over max cores""" compute1 = service.Service('host1', @@ -168,6 +260,10 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -201,6 +297,10 @@ class SimpleDriverTestCase(test.TestCase): 'volume', FLAGS.volume_manager) volume2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-volume') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-volume') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -224,6 +324,10 @@ class SimpleDriverTestCase(test.TestCase): 'volume', FLAGS.volume_manager) volume2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-volume') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-volume') + db.service_update(self.context, s1['id'], {'disabled': False}) + db.service_update(self.context, s2['id'], {'disabled': False}) volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): From b483167572c2b3aa7aca890e28188e82a23e6c7f Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 22 Dec 2010 13:27:51 -0500 Subject: [PATCH 27/54] Adding myself and Antony Messerli to the Authors file --- Authors | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Authors b/Authors index fa38ef0b..6e873acf 100644 --- a/Authors +++ b/Authors @@ -1,9 +1,11 @@ Andy Smith Anne Gentle Anthony Young +Antony Messerli Armando Migliaccio Chris Behrens Chmouel Boudjnah +Cory Wright Dean Troyer Devin Carlen Ed Leafe From 06ebf66f9d750f0b5d9ad87998afbd05688cb1c2 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 22 Dec 2010 13:01:33 -0600 Subject: [PATCH 28/54] Getting Snapshots to work with cloudservers command-line tool --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 51956d5c..cf4614b8 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -214,7 +214,6 @@ DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') DEFINE_integer('glance_port', 9292, 'glance port') DEFINE_string('glance_host', '127.0.0.1', 'glance host') -DEFINE_string('glance_storage_location', 'swift://username:api_key@auth.api.rackspacecloud.com/v1.0/cloudservers', 'glance storage location') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') @@ -236,6 +235,7 @@ DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', 'Url to ec2 api server') +DEFINE_string('default_project', 'openstack', 'default project for openstack') DEFINE_string('default_image', 'ami-11111', 'default image to use, testing only') DEFINE_string('default_instance_type', 'm1.small', From 30f23c2b71c50ce5127faf8985b50167532576a3 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 22 Dec 2010 14:00:31 -0600 Subject: [PATCH 29/54] i18n support for xs-snaps --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index b61447c7..d773a7e4 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -213,7 +213,7 @@ DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') DEFINE_integer('glance_port', 9292, 'glance port') -DEFINE_string('glance_host', utils.get_my_ip(),, 'glance host') +DEFINE_string('glance_host', utils.get_my_ip(), 'glance host') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', utils.get_my_ip(), 's3 host (for infrastructure)') DEFINE_string('s3_dmz', utils.get_my_ip(), 's3 dmz ip (for instances)') From 326fb917eec31d5f77e2f451b032a907a8402cc2 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 22 Dec 2010 14:54:02 -0600 Subject: [PATCH 30/54] Adding more comments regarding XS snapshots --- Authors | 1 + nova/tests/compute_unittest.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/Authors b/Authors index 0b048bec..d353b58f 100644 --- a/Authors +++ b/Authors @@ -24,6 +24,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark +Rick Harris Ryan Lucio Sandy Walsh Soren Hansen diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 187ca31d..025291a9 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -142,6 +142,14 @@ class ComputeTestCase(test.TestCase): self.compute.reboot_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) + def test_snapshot(self): + """Ensure instance can be snapshotted""" + instance_id = self._create_instance() + name = "myfakesnapshot" + self.compute.run_instance(self.context, instance_id) + self.compute.snapshot_instance(self.context, instance_id, name) + self.compute.terminate_instance(self.context, instance_id) + def test_console_output(self): """Make sure we can get console output from instance""" instance_id = self._create_instance() From 7e9e072cbec3a8af79895f02a3dc645df699eb8b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 23 Dec 2010 01:32:57 +0000 Subject: [PATCH 31/54] fix commits from Anthony and Vish that were committed with the wrong email --- .mailmap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index 8041e234..c01f964d 100644 --- a/.mailmap +++ b/.mailmap @@ -24,7 +24,7 @@ + + # These are from people who failed to set a proper committer -. . -. From dfaffd1d21f93304fb9be76e2d27a504f1305cd5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 23 Dec 2010 02:03:39 +0000 Subject: [PATCH 32/54] Add Ryan Lane as well --- .mailmap | 3 +-- Authors | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index c01f964d..9ab7db74 100644 --- a/.mailmap +++ b/.mailmap @@ -26,5 +26,4 @@ -# These are from people who failed to set a proper committer -. + diff --git a/Authors b/Authors index 0b048bec..250b3b2a 100644 --- a/Authors +++ b/Authors @@ -24,6 +24,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark +Ryan Lane Ryan Lucio Sandy Walsh Soren Hansen From 9fbdc5afc3f7883ee3524c9510b088c23fe5a0a8 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Thu, 23 Dec 2010 20:35:16 +0000 Subject: [PATCH 33/54] added _() for gettext and a couple of pep8s --- nova/tests/compute_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 111a43cd..14954c3a 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -100,13 +100,13 @@ class ComputeTestCase(test.TestCase): self.compute.run_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) - logging.info("Running instances: %s", instances) + logging.info(_("Running instances: %s"), instances) self.assertEqual(len(instances), 1) self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) - logging.info("After terminating instances: %s", instances) + logging.info(_("After terminating instances: %s"), instances) self.assertEqual(len(instances), 0) def test_run_terminate_timestamps(self): From 3109d1cb3ad09a5510235bca15e2abd70a9d0ed6 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Fri, 24 Dec 2010 19:17:02 +0000 Subject: [PATCH 34/54] remove requirement of sudo on tests --- nova/tests/test_xenapi.py | 1 + nova/tests/xenapi/stubs.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index b5d3ea39..ed2e4ffd 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -48,6 +48,7 @@ class XenAPIVolumeTestCase(test.TestCase): FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' fakes.stub_out_db_instance_api(self.stubs) + stubs.stub_out_get_target(self.stubs) fake.reset() self.values = {'name': 1, 'id': 1, 'project_id': 'fake', diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 1dacad6a..a7e592fe 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -18,12 +18,13 @@ from nova.virt import xenapi_conn from nova.virt.xenapi import fake +from nova.virt.xenapi import volume_utils def stubout_session(stubs, cls): - """ Stubs out two methods from XenAPISession """ + """Stubs out two methods from XenAPISession""" def fake_import(self): - """ Stubs out get_imported_xenapi of XenAPISession """ + """Stubs out get_imported_xenapi of XenAPISession""" fake_module = 'nova.virt.xenapi.fake' from_list = ['fake'] return __import__(fake_module, globals(), locals(), from_list, -1) @@ -34,6 +35,14 @@ def stubout_session(stubs, cls): fake_import) +def stub_out_get_target(stubs): + """Stubs out _get_target in volume_utils""" + def fake_get_target(volume_id): + return (None, None) + + stubs.Set(volume_utils, '_get_target', fake_get_target) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): From d3ad0918c257293a06de0f1147ca54e29e005fcb Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Tue, 28 Dec 2010 01:37:04 +0000 Subject: [PATCH 35/54] Bug #694880: nova-compute now depends upon Cheetah even when not using libvirt Only import Cheetah when needed, as we do already with libvirt and libxml2. This ensures that users of other virt backends don't need Cheetah to run nova-compute. --- nova/tests/test_virt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 1c155abe..4aa489d0 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -33,6 +33,7 @@ flags.DECLARE('instances_path', 'nova.compute.manager') class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() + libvirt_conn._late_load_cheetah() self.flags(fake_call=True) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake', From 2316cfb708a4b448d13502606ed19f31e0ab187c Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 28 Dec 2010 01:40:24 -0800 Subject: [PATCH 36/54] Output of run_tests.sh to be closer to trial --- run_tests.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 run_tests.py diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 00000000..56a8bffe --- /dev/null +++ b/run_tests.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import unittest +import sys + +from nose import config +from nose import result +from nose import core + + +class NovaTestResult(result.TextTestResult): + def __init__(self, *args, **kw): + result.TextTestResult.__init__(self, *args, **kw) + self._last_case = None + + def getDescription(self, test): + return str(test) + + def startTest(self, test): + unittest.TestResult.startTest(self, test) + current_case = test.test.__class__.__name__ + + if self.showAll: + if current_case != self._last_case: + self.stream.writeln(current_case) + self._last_case = current_case + + self.stream.write( + ' %s' % str(test.test._testMethodName).ljust(60)) + self.stream.flush() + + +class NovaTestRunner(core.TextTestRunner): + def _makeResult(self): + return NovaTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config) + + +if __name__ == '__main__': + c = config.Config(stream=sys.stdout, + env=os.environ, + verbosity=3) + + runner = NovaTestRunner(stream=c.stream, + verbosity=c.verbosity, + config=c) + sys.exit(core.run(config=c, testRunner=runner)) From 270c7ad370e7b6c8711fd6b17552ef0fa16d34b6 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 28 Dec 2010 12:53:32 -0600 Subject: [PATCH 37/54] Add some basic snapshot tests --- nova/tests/test_xenapi.py | 108 +++++++++++++++++++++++++------------ nova/tests/xenapi/stubs.py | 66 +++++++++++++++++++++++ 2 files changed, 140 insertions(+), 34 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index ed2e4ffd..f0d84e9a 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -29,9 +29,9 @@ from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state from nova.virt import xenapi_conn -from nova.virt.xenapi import fake +from nova.virt.xenapi import fake as xenapi_fake from nova.virt.xenapi import volume_utils -from nova.tests.db import fakes +from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs FLAGS = flags.FLAGS @@ -47,9 +47,9 @@ class XenAPIVolumeTestCase(test.TestCase): FLAGS.target_host = '127.0.0.1' FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' - fakes.stub_out_db_instance_api(self.stubs) + db_fakes.stub_out_db_instance_api(self.stubs) stubs.stub_out_get_target(self.stubs) - fake.reset() + xenapi_fake.reset() self.values = {'name': 1, 'id': 1, 'project_id': 'fake', 'user_id': 'fake', @@ -83,7 +83,7 @@ class XenAPIVolumeTestCase(test.TestCase): label = 'SR-%s' % vol['ec2_id'] description = 'Test-SR' sr_ref = helper.create_iscsi_storage(session, info, label, description) - srs = fake.get_all('SR') + srs = xenapi_fake.get_all('SR') self.assertEqual(sr_ref, srs[0]) db.volume_destroy(context.get_admin_context(), vol['id']) @@ -107,17 +107,17 @@ class XenAPIVolumeTestCase(test.TestCase): conn = xenapi_conn.get_connection(False) volume = self._create_volume() instance = db.instance_create(self.values) - fake.create_vm(instance.name, 'Running') + xenapi_fake.create_vm(instance.name, 'Running') result = conn.attach_volume(instance.name, volume['ec2_id'], '/dev/sdc') def check(): # check that the VM has a VBD attached to it # Get XenAPI reference for the VM - vms = fake.get_all('VM') + vms = xenapi_fake.get_all('VM') # Get XenAPI record for VBD - vbds = fake.get_all('VBD') - vbd = fake.get_record('VBD', vbds[0]) + vbds = xenapi_fake.get_all('VBD') + vbd = xenapi_fake.get_record('VBD', vbds[0]) vm_ref = vbd['VM'] self.assertEqual(vm_ref, vms[0]) @@ -130,7 +130,7 @@ class XenAPIVolumeTestCase(test.TestCase): conn = xenapi_conn.get_connection(False) volume = self._create_volume() instance = db.instance_create(self.values) - fake.create_vm(instance.name, 'Running') + xenapi_fake.create_vm(instance.name, 'Running') self.assertRaises(Exception, conn.attach_volume, instance.name, @@ -156,41 +156,66 @@ class XenAPIVMTestCase(test.TestCase): self.stubs = stubout.StubOutForTesting() FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' - fake.reset() - fakes.stub_out_db_instance_api(self.stubs) - fake.create_network('fake', FLAGS.flat_network_bridge) + xenapi_fake.reset() + db_fakes.stub_out_db_instance_api(self.stubs) + xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + self.conn = xenapi_conn.get_connection(False) def test_list_instances_0(self): - stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - conn = xenapi_conn.get_connection(False) - instances = conn.list_instances() + instances = self.conn.list_instances() self.assertEquals(instances, []) - def test_spawn(self): - stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - values = {'name': 1, 'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, - 'image_id': 1, - 'kernel_id': 2, - 'ramdisk_id': 3, - 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff', - } - conn = xenapi_conn.get_connection(False) - instance = db.instance_create(values) - conn.spawn(instance) + def test_instance_snapshot(self): + stubs.stubout_instance_snapshot(self.stubs) + instance = self._create_instance() + + name = "MySnapshot" + template_vm_ref = self.conn.snapshot(instance, name) + + def ensure_vm_was_torn_down(): + vm_labels = [] + for vm_ref in xenapi_fake.get_all('VM'): + vm_rec = xenapi_fake.get_record('VM', vm_ref) + if not vm_rec["is_control_domain"]: + vm_labels.append(vm_rec["name_label"]) + + self.assertEquals(vm_labels, [1]) + + def ensure_vbd_was_torn_down(): + vbd_labels = [] + for vbd_ref in xenapi_fake.get_all('VBD'): + vbd_rec = xenapi_fake.get_record('VBD', vbd_ref) + vbd_labels.append(vbd_rec["vm_name_label"]) + + self.assertEquals(vbd_labels, [1]) + + def ensure_vdi_was_torn_down(): + for vdi_ref in xenapi_fake.get_all('VDI'): + vdi_rec = xenapi_fake.get_record('VDI', vdi_ref) + name_label = vdi_rec["name_label"] + self.assert_(not name_label.endswith('snapshot')) def check(): - instances = conn.list_instances() + ensure_vm_was_torn_down() + ensure_vbd_was_torn_down() + ensure_vdi_was_torn_down() + + check() + + def test_spawn(self): + instance = self._create_instance() + + def check(): + instances = self.conn.list_instances() self.assertEquals(instances, [1]) # Get Nova record for VM - vm_info = conn.get_info(1) + vm_info = self.conn.get_info(1) # Get XenAPI record for VM - vms = fake.get_all('VM') - vm = fake.get_record('VM', vms[0]) + vms = xenapi_fake.get_all('VM') + vm = xenapi_fake.get_record('VM', vms[0]) # Check that m1.large above turned into the right thing. instance_type = instance_types.INSTANCE_TYPES['m1.large'] @@ -218,3 +243,18 @@ class XenAPIVMTestCase(test.TestCase): self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.stubs.UnsetAll() + + def _create_instance(self): + """Creates and spawns a test instance""" + values = {'name': 1, 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + instance = db.instance_create(values) + self.conn.spawn(instance) + return instance diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index a7e592fe..55f751f1 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -19,6 +19,54 @@ from nova.virt import xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils +from nova.virt.xenapi import vm_utils + + +def stubout_instance_snapshot(stubs): + @classmethod + def fake_fetch_image(cls, session, instance_id, image, user, project, + type): + # Stubout wait_for_task + def fake_wait_for_task(self, id, task): + class FakeEvent: + + def send(self, value): + self.rv = value + + def wait(self): + return self.rv + + done = FakeEvent() + self._poll_task(id, task, done) + rv = done.wait() + return rv + + stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', + fake_wait_for_task) + + from nova.virt.xenapi.fake import create_vdi + name_label = "instance-%s" % instance_id + #TODO: create fake SR record + sr_ref = "fakesr" + vdi_ref = create_vdi(name_label=name_label, read_only=False, + sr_ref=sr_ref, sharable=False) + vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) + vdi_uuid = vdi_rec['uuid'] + return vdi_uuid + + stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) + + def fake_parse_xmlrpc_value(val): + return val + + stubs.Set(xenapi_conn, '_parse_xmlrpc_value', fake_parse_xmlrpc_value) + + def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, + original_parent_uuid): + #TODO(sirp): Should we actually fake out the data here + return "fakeparent" + + stubs.Set(vm_utils, 'wait_for_vhd_coalesce', fake_wait_for_vhd_coalesce) def stubout_session(stubs, cls): @@ -63,6 +111,24 @@ class FakeSessionForVMTests(fake.SessionBase): vm['is_a_template'] = False vm['is_control_domain'] = False + def VM_snapshot(self, session_ref, vm_ref, label): + status = "Running" + template_vm_ref = fake.create_vm(label, status, is_a_template=True, + is_control_domain=False) + + sr_ref = "fakesr" + template_vdi_ref = fake.create_vdi(label, read_only=True, + sr_ref=sr_ref, sharable=False) + + template_vbd_ref = fake.create_vbd(template_vm_ref, template_vdi_ref) + return template_vm_ref + + def VDI_destroy(self, session_ref, vdi_ref): + fake.destroy_vdi(vdi_ref) + + def VM_destroy(self, session_ref, vm_ref): + fake.destroy_vm(vm_ref) + class FakeSessionForVolumeTests(fake.SessionBase): """ Stubs out a XenAPISession for Volume tests """ From 8472bc47200c8300147797cc26e58331790d1b74 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Tue, 28 Dec 2010 16:35:56 -0500 Subject: [PATCH 38/54] Update .mailmap with both email addresses for Ant and myself --- .mailmap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mailmap b/.mailmap index 9ab7db74..9e7fb1ec 100644 --- a/.mailmap +++ b/.mailmap @@ -27,3 +27,5 @@ + + From dd82ba1c2c2f5e4d47992b5f8b70a3350332c2a4 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 28 Dec 2010 17:54:31 -0500 Subject: [PATCH 39/54] Defualt services to enabled. --- nova/tests/test_scheduler.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 65430fb3..e8021ed5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -133,6 +133,10 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': True}) + db.service_update(self.context, s2['id'], {'disabled': True}) hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(0, len(hosts)) compute1.kill() @@ -152,10 +156,6 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(2, len(hosts)) compute1.kill() @@ -173,10 +173,6 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -200,10 +196,6 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance(availability_zone='nova:host1') @@ -225,8 +217,7 @@ class SimpleDriverTestCase(test.TestCase): now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) past = now - delta - db.service_update(self.context, s1['id'], {'disabled': False, - 'updated_at': past}) + db.service_update(self.context, s1['id'], {'updated_at': past}) instance_id2 = self._create_instance(availability_zone='nova:host1') self.assertRaises(driver.WillNotSchedule, self.scheduler.driver.schedule_run_instance, @@ -241,7 +232,8 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute1.start() - db.service_get_by_args(self.context, 'host1', 'nova-compute') + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': True}) instance_id2 = self._create_instance(availability_zone='nova:host1') host = self.scheduler.driver.schedule_run_instance(self.context, instance_id2) @@ -261,10 +253,6 @@ class SimpleDriverTestCase(test.TestCase): 'compute', FLAGS.compute_manager) compute2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -298,10 +286,6 @@ class SimpleDriverTestCase(test.TestCase): 'volume', FLAGS.volume_manager) volume2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-volume') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-volume') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -325,10 +309,6 @@ class SimpleDriverTestCase(test.TestCase): 'volume', FLAGS.volume_manager) volume2.start() - s1 = db.service_get_by_args(self.context, 'host1', 'nova-volume') - s2 = db.service_get_by_args(self.context, 'host2', 'nova-volume') - db.service_update(self.context, s1['id'], {'disabled': False}) - db.service_update(self.context, s2['id'], {'disabled': False}) volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): From 9b58e47aaf8989803a228a660670b337a5c14f8b Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 29 Dec 2010 10:35:29 -0600 Subject: [PATCH 40/54] Updating Authors --- .mailmap | 1 + Authors | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 9ab7db74..9ac8f2ef 100644 --- a/.mailmap +++ b/.mailmap @@ -27,3 +27,4 @@ + diff --git a/Authors b/Authors index 299114da..407b407c 100644 --- a/Authors +++ b/Authors @@ -25,7 +25,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark -Rick Harris +Rick Harris Ryan Lane Ryan Lucio Salvatore Orlando From 50e54140b09770ab4d6de0339c988879aa0b96a5 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 29 Dec 2010 14:02:57 -0600 Subject: [PATCH 41/54] Added tests --- nova/tests/test_compute.py | 16 ++++++++++++++-- nova/tests/test_xenapi.py | 30 ++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index bcb8a152..757b1f46 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -102,13 +102,15 @@ class ComputeTestCase(test.TestCase): instances = db.instance_get_all(context.get_admin_context()) logging.info(_("Running instances: %s"), instances) - self.assertEqual(len(instances), 1) + + instance_count = len(instances) + self.assertNotEqual(instance_count, 0) self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info(_("After terminating instances: %s"), instances) - self.assertEqual(len(instances), 0) + self.assertEqual(instance_count, len(instances) + 1) def test_run_terminate_timestamps(self): """Make sure timestamps are set for launched and destroyed""" @@ -151,6 +153,16 @@ class ComputeTestCase(test.TestCase): self.compute.reboot_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) + def test_diagnostics(self): + """Ensure instance diagnostics are available""" + instance_id = self._create_instance() + self.compute.get_diagnostics(self.context, instance_id) + + def test_actions(self): + """Ensure instance actions are available""" + instance_id = self._create_instance() + self.compute.get_actions(self.context, instance_id) + def test_console_output(self): """Make sure we can get console output from instance""" instance_id = self._create_instance() diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index ed2e4ffd..11baead8 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -159,6 +159,16 @@ class XenAPIVMTestCase(test.TestCase): fake.reset() fakes.stub_out_db_instance_api(self.stubs) fake.create_network('fake', FLAGS.flat_network_bridge) + self.values = { + "name": 1, + "id": 1, + "project_id": self.project.id, + "user_id": self.user.id, + "image_id": 1, + "kernel_id": 2, + "ramdisk_id": 3, + "instance_type": "m1.large", + "mac_address": "aa:bb:cc:dd:ee:ff"} def test_list_instances_0(self): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) @@ -166,19 +176,19 @@ class XenAPIVMTestCase(test.TestCase): instances = conn.list_instances() self.assertEquals(instances, []) + def test_get_diagnostics(self): + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + conn = xenapi_conn.get_connection(False) + + instance = db.instance_create(self.values) + conn.spawn(instance) + + conn.get_diagnostics(instance) + def test_spawn(self): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - values = {'name': 1, 'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, - 'image_id': 1, - 'kernel_id': 2, - 'ramdisk_id': 3, - 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff', - } conn = xenapi_conn.get_connection(False) - instance = db.instance_create(values) + instance = db.instance_create(self.values) conn.spawn(instance) def check(): From 7fef5bb18f424f2cb61da13c65081678ceee2166 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 29 Dec 2010 17:01:34 -0500 Subject: [PATCH 43/54] Fix scheduler testcase so it knows all flags and can run in isolation. --- nova/tests/test_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 91517cc5..78e4a1c7 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -33,6 +33,7 @@ from nova.scheduler import driver FLAGS = flags.FLAGS flags.DECLARE('max_cores', 'nova.scheduler.simple') +flags.DECLARE('stub_network', 'nova.compute.manager') class TestDriver(driver.Scheduler): From a8268a3a7492a721575354633c134c3fc7f9cb44 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 29 Dec 2010 17:08:42 -0500 Subject: [PATCH 44/54] Pep-8 cleanup. --- bin/nova-api-paste | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/nova-api-paste b/bin/nova-api-paste index dcb76522..3d26fdb4 100755 --- a/bin/nova-api-paste +++ b/bin/nova-api-paste @@ -48,6 +48,7 @@ FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'openstack'] + def load_configuration(paste_config): """Load the paste configuration from the config file and return it.""" config = None @@ -68,12 +69,14 @@ def load_configuration(paste_config): print "Paste config at %s has no secion for any known apis" % paste_config os.exit(1) + def launch_api(paste_config_file, section, server, port, host): """Launch an api server from the specified port and IP.""" LOG.debug("Launching api %s on %s:%s", section, host, port) app = deploy.loadapp('config:%s' % paste_config_file, name=section) server.start(app, int(port), host) + def run_app(paste_config_file): LOG.debug("Using paste.deploy config at: %s", configfile) config = load_configuration(paste_config_file) @@ -89,6 +92,7 @@ def run_app(paste_config_file): LOG.debug("All api servers launched, now waiting") server.wait() + if __name__ == '__main__': FLAGS(sys.argv) configfiles = ['/etc/nova/nova-api.conf'] From cbe318e8b9acaeba98395a7a3d36f034b4785e7c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 29 Dec 2010 17:15:50 -0500 Subject: [PATCH 45/54] i18n --- bin/nova-api-paste | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bin/nova-api-paste b/bin/nova-api-paste index 3d26fdb4..9bcb9837 100755 --- a/bin/nova-api-paste +++ b/bin/nova-api-paste @@ -27,9 +27,6 @@ import sys from paste import deploy -from nova import flags -from nova import wsgi - # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -40,6 +37,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import wsgi + LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) LOG.addHandler(logging.StreamHandler()) @@ -65,22 +65,24 @@ def load_configuration(paste_config): if FLAGS.verbose: logging.getLogger().setLevel(logging.DEBUG) return config - LOG.debug("Paste config at %s has no secion for known apis", paste_config) - print "Paste config at %s has no secion for any known apis" % paste_config + LOG.debug(_("Paste config at %s has no secion for known apis"), + paste_config) + print _("Paste config at %s has no secion for any known apis") % \ + paste_config os.exit(1) def launch_api(paste_config_file, section, server, port, host): """Launch an api server from the specified port and IP.""" - LOG.debug("Launching api %s on %s:%s", section, host, port) + LOG.debug(_("Launching %s api on %s:%s"), section, host, port) app = deploy.loadapp('config:%s' % paste_config_file, name=section) server.start(app, int(port), host) def run_app(paste_config_file): - LOG.debug("Using paste.deploy config at: %s", configfile) + LOG.debug(_("Using paste.deploy config at: %s"), configfile) config = load_configuration(paste_config_file) - LOG.debug("Configuration: %r", config) + LOG.debug(_("Configuration: %r"), config) server = wsgi.Server() ip = config.get('host', None) for api in API_ENDPOINTS: @@ -89,7 +91,7 @@ def run_app(paste_config_file): continue host = config.get("%s_host" % api, None) or ip or '0.0.0.0' launch_api(configfile, api, server, port, host) - LOG.debug("All api servers launched, now waiting") + LOG.debug(_("All api servers launched, now waiting")) server.wait() @@ -104,4 +106,4 @@ if __name__ == '__main__': run_app(configfile) break else: - LOG.debug("Skipping missing configuration: %s", configfile) + LOG.debug(_("Skipping missing configuration: %s"), configfile) From 8a5569beb926ce51f504c0016c61989e7eee5d15 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 30 Dec 2010 01:19:38 -0500 Subject: [PATCH 46/54] Clean up how we determine IP to bind to. --- bin/nova-api-paste | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-api-paste b/bin/nova-api-paste index 9bcb9837..6ee833a1 100755 --- a/bin/nova-api-paste +++ b/bin/nova-api-paste @@ -84,12 +84,12 @@ def run_app(paste_config_file): config = load_configuration(paste_config_file) LOG.debug(_("Configuration: %r"), config) server = wsgi.Server() - ip = config.get('host', None) + ip = config.get('host', '0.0.0.0') for api in API_ENDPOINTS: port = config.get("%s_port" % api, None) if not port: continue - host = config.get("%s_host" % api, None) or ip or '0.0.0.0' + host = config.get("%s_host" % api, ip) launch_api(configfile, api, server, port, host) LOG.debug(_("All api servers launched, now waiting")) server.wait() From 2bac387b37ed58f1a4afa5f240f27ad9508597a9 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 30 Dec 2010 13:31:56 -0800 Subject: [PATCH 47/54] change exit code --- run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 56a8bffe..5b8617f6 100644 --- a/run_tests.py +++ b/run_tests.py @@ -65,4 +65,4 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner)) From 7452f91a0746f0ee5a66139994b377fa275672d1 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 30 Dec 2010 16:06:48 -0600 Subject: [PATCH 48/54] Review feedback --- nova/tests/test_compute.py | 11 ++--------- nova/tests/test_xenapi.py | 31 ++++++------------------------- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e3679b6b..2664adc0 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -102,15 +102,13 @@ class ComputeTestCase(test.TestCase): instances = db.instance_get_all(context.get_admin_context()) logging.info(_("Running instances: %s"), instances) - - instance_count = len(instances) - self.assertNotEqual(instance_count, 0) + self.assertEqual(len(instances), 1) self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info(_("After terminating instances: %s"), instances) - self.assertEqual(instance_count, len(instances) + 1) + self.assertEqual(len(instances), 0) def test_run_terminate_timestamps(self): """Make sure timestamps are set for launched and destroyed""" @@ -158,11 +156,6 @@ class ComputeTestCase(test.TestCase): instance_id = self._create_instance() self.compute.get_diagnostics(self.context, instance_id) - def test_actions(self): - """Ensure instance actions are available""" - instance_id = self._create_instance() - self.compute.get_actions(self.context, instance_id) - def test_snapshot(self): """Ensure instance can be snapshotted""" instance_id = self._create_instance() diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 3528d7bf..a0975067 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -156,10 +156,11 @@ class XenAPIVMTestCase(test.TestCase): self.stubs = stubout.StubOutForTesting() FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' - - fake.reset() - fakes.stub_out_db_instance_api(self.stubs) - fake.create_network('fake', FLAGS.flat_network_bridge) + xenapi_fake.reset() + db_fakes.stub_out_db_instance_api(self.stubs) + xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + self.conn = xenapi_conn.get_connection(False) self.values = { "name": 1, "id": 1, @@ -171,12 +172,6 @@ class XenAPIVMTestCase(test.TestCase): "instance_type": "m1.large", "mac_address": "aa:bb:cc:dd:ee:ff"} - xenapi_fake.reset() - db_fakes.stub_out_db_instance_api(self.stubs) - xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) - stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - self.conn = xenapi_conn.get_connection(False) - def test_list_instances_0(self): instances = self.conn.list_instances() self.assertEquals(instances, []) @@ -228,11 +223,6 @@ class XenAPIVMTestCase(test.TestCase): check() def test_spawn(self): - stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - conn = xenapi_conn.get_connection(False) - instance = db.instance_create(self.values) - conn.spawn(instance) - instance = self._create_instance() def check(): @@ -275,15 +265,6 @@ class XenAPIVMTestCase(test.TestCase): def _create_instance(self): """Creates and spawns a test instance""" - values = {'name': 1, 'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, - 'image_id': 1, - 'kernel_id': 2, - 'ramdisk_id': 3, - 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff', - } - instance = db.instance_create(values) + instance = db.instance_create(self.values) self.conn.spawn(instance) return instance From a51d36b60631d351592565f96e936bd9bb0b77ce Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 30 Dec 2010 18:12:10 -0600 Subject: [PATCH 49/54] Removed problematic test --- nova/tests/test_compute.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 2664adc0..1fb9143f 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -151,11 +151,6 @@ class ComputeTestCase(test.TestCase): self.compute.reboot_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) - def test_diagnostics(self): - """Ensure instance diagnostics are available""" - instance_id = self._create_instance() - self.compute.get_diagnostics(self.context, instance_id) - def test_snapshot(self): """Ensure instance can be snapshotted""" instance_id = self._create_instance() From 803c0e37c6b263943bbf30ae37e789d2f4a208b3 Mon Sep 17 00:00:00 2001 From: Ryan Lucio Date: Thu, 30 Dec 2010 16:21:11 -0800 Subject: [PATCH 50/54] Converted the pool_recycle setting to be a flag with a default of 3600 seconds --- nova/flags.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 76a98d35..338dcbf4 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -260,6 +260,9 @@ DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), DEFINE_string('sql_connection', 'sqlite:///$state_path/nova.sqlite', 'connection string for sql database') +DEFINE_string('sql_idle_timeout', + '3600', + 'timeout for idle sql database connections') DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', 'Manager for compute') From 78e6037a5e82cdd3b410e1b7f2e3889fc6fd7188 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 30 Dec 2010 18:56:12 -0600 Subject: [PATCH 51/54] Improved test --- nova/tests/test_xenapi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index a0975067..539d132b 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -180,9 +180,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) conn = xenapi_conn.get_connection(False) - instance = db.instance_create(self.values) - conn.spawn(instance) - + instance = self._create_instance() conn.get_diagnostics(instance) def test_instance_snapshot(self): From 298516e480d9dcffd9665cbd79a837d12aa2654e Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 30 Dec 2010 19:07:20 -0600 Subject: [PATCH 52/54] Cleanup --- nova/tests/test_xenapi.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 539d132b..c95a53af 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -161,27 +161,14 @@ class XenAPIVMTestCase(test.TestCase): xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) self.conn = xenapi_conn.get_connection(False) - self.values = { - "name": 1, - "id": 1, - "project_id": self.project.id, - "user_id": self.user.id, - "image_id": 1, - "kernel_id": 2, - "ramdisk_id": 3, - "instance_type": "m1.large", - "mac_address": "aa:bb:cc:dd:ee:ff"} def test_list_instances_0(self): instances = self.conn.list_instances() self.assertEquals(instances, []) def test_get_diagnostics(self): - stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) - conn = xenapi_conn.get_connection(False) - instance = self._create_instance() - conn.get_diagnostics(instance) + self.conn.get_diagnostics(instance) def test_instance_snapshot(self): stubs.stubout_instance_snapshot(self.stubs) @@ -263,6 +250,15 @@ class XenAPIVMTestCase(test.TestCase): def _create_instance(self): """Creates and spawns a test instance""" - instance = db.instance_create(self.values) + values = {'name': 1, 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff' + } + instance = db.instance_create(values) self.conn.spawn(instance) return instance From ed5e3ac8bbf49cd5c7d88795654e53eb466b5308 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 30 Dec 2010 20:03:21 -0800 Subject: [PATCH 53/54] Ignore CA/crl.pem --- .bzrignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.bzrignore b/.bzrignore index 82db46fa..d81a7d82 100644 --- a/.bzrignore +++ b/.bzrignore @@ -6,6 +6,7 @@ keys networks nova.sqlite CA/cacert.pem +CA/crl.pem CA/index.txt* CA/openssl.cnf CA/serial* From 5aba86baffcc78b76ec5e7b7981ba4bb21fa2fed Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 3 Jan 2011 16:08:52 -0600 Subject: [PATCH 54/54] Fixes LP688545 --- nova/flags.py | 2 ++ nova/tests/test_xenapi.py | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index e872ba21..4b733492 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -266,6 +266,8 @@ DEFINE_string('sql_connection', DEFINE_string('sql_idle_timeout', '3600', 'timeout for idle sql database connections') +DEFINE_integer('sql_max_retries', 12, 'sql connection attempts') +DEFINE_integer('sql_retry_interval', 10, 'sql connection retry interval') DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', 'Manager for compute') diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index c95a53af..33571dad 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -250,15 +250,16 @@ class XenAPIVMTestCase(test.TestCase): def _create_instance(self): """Creates and spawns a test instance""" - values = {'name': 1, 'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, - 'image_id': 1, - 'kernel_id': 2, - 'ramdisk_id': 3, - 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff' - } + values = { + 'name': 1, + 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff'} instance = db.instance_create(values) self.conn.spawn(instance) return instance