From 622fe886723491ff4dde0b0fa5fd7801ebee273c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 17 Jan 2011 19:40:35 -0500 Subject: [PATCH 001/107] Stubbed-out code for working with provider-firewalls. --- nova/adminclient.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/nova/adminclient.py b/nova/adminclient.py index b2609c8c4..90ad4d9dd 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -190,6 +190,22 @@ class HostInfo(object): setattr(self, name, value) +class SimpleResponse(object): + def __init__(self, connection=None): + self.connection = connection + self.status = None + self.message = '' + + def __repr__(self): + return 'Status:%s' % self.status + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name.lower(), str(value)) + + class NovaAdminClient(object): def __init__( @@ -373,3 +389,8 @@ class NovaAdminClient(object): def get_hosts(self): return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) + + def block_ips(self, cidr): + """Block incoming traffic from specified hosts.""" + return self.apiconn.get_object('BlockExternalAddresses', + {'Cidr': cidr}, SimpleResponse) From f9b7ab4e75d5eb4b4a14feb69d1ce5cce0591ae1 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 19 Jan 2011 15:16:12 -0500 Subject: [PATCH 002/107] Whitespace (pep8) cleanups. --- nova/adminclient.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 90ad4d9dd..0cdb8c6fb 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -190,20 +190,20 @@ class HostInfo(object): setattr(self, name, value) -class SimpleResponse(object): - def __init__(self, connection=None): - self.connection = connection - self.status = None +class SimpleResponse(object): + def __init__(self, connection=None): + self.connection = connection + self.status = None self.message = '' - - def __repr__(self): - return 'Status:%s' % self.status - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - setattr(self, name.lower(), str(value)) + + def __repr__(self): + return 'Status:%s' % self.status + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name.lower(), str(value)) class NovaAdminClient(object): @@ -224,8 +224,7 @@ class NovaAdminClient(object): self.apiconn = boto.connect_ec2(aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=parts['is_secure'], - region=RegionInfo(None, - region, + region=RegionInfo(None, region, parts['ip']), port=parts['port'], path='/services/Admin', From ad91f9e213bd018f2103b31a64e644b251624d37 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 26 Jan 2011 22:54:39 -0800 Subject: [PATCH 003/107] A couple of bugfixes. --- nova/adminclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 9b43995fe..9652dcb9c 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -230,7 +230,8 @@ class InstanceType(object): class SimpleResponse(object): - def __init__(self): + def __init__(self, connection=None): + self.connection = connection self.status = None self.message = '' From 36c3eb523cc00b82397f1602f9105a3088d0d519 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 26 Jan 2011 22:56:34 -0800 Subject: [PATCH 004/107] Apply lp:707675 to this branch to be able to test. --- nova/tests/test_virt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 0b9b847a0..1008f32ae 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -221,7 +221,12 @@ class IptablesFirewallTestCase(test.TestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) - self.fw = libvirt_conn.IptablesFirewallDriver() + + class FakeLibvirtConnection(object): + pass + self.fake_libvirt_connection = FakeLibvirtConnection() + self.fw = libvirt_conn.IptablesFirewallDriver( + get_connection=lambda: self.fake_libvirt_connection) def tearDown(self): self.manager.delete_project(self.project) From 8eae885e7144e6a6d888c3cf86e0c6e3f091d83e Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 5 Apr 2011 15:17:17 -0400 Subject: [PATCH 005/107] Remove file leftover from conflict. --- nova/adminclient.py.THIS | 451 --------------------------------------- 1 file changed, 451 deletions(-) delete mode 100644 nova/adminclient.py.THIS diff --git a/nova/adminclient.py.THIS b/nova/adminclient.py.THIS deleted file mode 100644 index c96d6e1fe..000000000 --- a/nova/adminclient.py.THIS +++ /dev/null @@ -1,451 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Nova User API client library. -""" - -import base64 -import boto -import boto.exception -import httplib - -from boto.ec2.regioninfo import RegionInfo - - -DEFAULT_CLC_URL = 'http://127.0.0.1:8773' -DEFAULT_REGION = 'nova' - - -class UserInfo(object): - """ - Information about a Nova user, as parsed through SAX. - - **Fields Include** - - * username - * accesskey - * secretkey - * file (optional) containing zip of X509 cert & rc file - - """ - - def __init__(self, connection=None, username=None, endpoint=None): - self.connection = connection - self.username = username - self.endpoint = endpoint - - def __repr__(self): - return 'UserInfo:%s' % self.username - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'username': - self.username = str(value) - elif name == 'file': - self.file = base64.b64decode(str(value)) - elif name == 'accesskey': - self.accesskey = str(value) - elif name == 'secretkey': - self.secretkey = str(value) - - -class UserRole(object): - """ - Information about a Nova user's role, as parsed through SAX. - - **Fields include** - - * role - - """ - - def __init__(self, connection=None): - self.connection = connection - self.role = None - - def __repr__(self): - return 'UserRole:%s' % self.role - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'role': - self.role = value - else: - setattr(self, name, str(value)) - - -class ProjectInfo(object): - """ - Information about a Nova project, as parsed through SAX. - - **Fields include** - - * projectname - * description - * projectManagerId - * memberIds - - """ - - def __init__(self, connection=None): - self.connection = connection - self.projectname = None - self.description = None - self.projectManagerId = None - self.memberIds = [] - - def __repr__(self): - return 'ProjectInfo:%s' % self.projectname - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'projectname': - self.projectname = value - elif name == 'description': - self.description = value - elif name == 'projectManagerId': - self.projectManagerId = value - elif name == 'memberId': - self.memberIds.append(value) - else: - setattr(self, name, str(value)) - - -class ProjectMember(object): - """ - Information about a Nova project member, as parsed through SAX. - - **Fields include** - - * memberId - - """ - - def __init__(self, connection=None): - self.connection = connection - self.memberId = None - - def __repr__(self): - return 'ProjectMember:%s' % self.memberId - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'member': - self.memberId = value - else: - setattr(self, name, str(value)) - - -class HostInfo(object): - """ - Information about a Nova Host, as parsed through SAX. - - **Fields Include** - - * Disk stats - * Running Instances - * Memory stats - * CPU stats - * Network address info - * Firewall info - * Bridge and devices - - """ - - def __init__(self, connection=None): - self.connection = connection - self.hostname = None - - def __repr__(self): - return 'Host:%s' % self.hostname - - # this is needed by the sax parser, so ignore the ugly name - def startElement(self, name, attrs, connection): - return None - - # this is needed by the sax parser, so ignore the ugly name - def endElement(self, name, value, connection): - setattr(self, name, value) - - -class InstanceType(object): - """ - Information about a Nova instance type, as parsed through SAX. - - **Fields include** - - * name - * vcpus - * disk_gb - * memory_mb - * flavor_id - - """ - - def __init__(self, connection=None): - self.connection = connection - self.name = None - self.vcpus = None - self.disk_gb = None - self.memory_mb = None - self.flavor_id = None - - def __repr__(self): - return 'InstanceType:%s' % self.name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == "memoryMb": - self.memory_mb = str(value) - elif name == "flavorId": - self.flavor_id = str(value) - elif name == "diskGb": - self.disk_gb = str(value) - else: - setattr(self, name, str(value)) - - -class SimpleResponse(object): - def __init__(self, connection=None): - self.connection = connection - self.status = None - self.message = '' - - def __repr__(self): - return 'Status:%s' % self.status - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - setattr(self, name.lower(), str(value)) - - -class NovaAdminClient(object): - - def __init__( - self, - clc_url=DEFAULT_CLC_URL, - region=DEFAULT_REGION, - access_key=None, - secret_key=None, - **kwargs): - parts = self.split_clc_url(clc_url) - - self.clc_url = clc_url - self.region = region - self.access = access_key - self.secret = secret_key - self.apiconn = boto.connect_ec2(aws_access_key_id=access_key, - aws_secret_access_key=secret_key, - is_secure=parts['is_secure'], - region=RegionInfo(None, region, - parts['ip']), - port=parts['port'], - path='/services/Admin', - **kwargs) - self.apiconn.APIVersion = 'nova' - - def connection_for(self, username, project, clc_url=None, region=None, - **kwargs): - """Returns a boto ec2 connection for the given username.""" - if not clc_url: - clc_url = self.clc_url - if not region: - region = self.region - parts = self.split_clc_url(clc_url) - user = self.get_user(username) - access_key = '%s:%s' % (user.accesskey, project) - return boto.connect_ec2(aws_access_key_id=access_key, - aws_secret_access_key=user.secretkey, - is_secure=parts['is_secure'], - region=RegionInfo(None, - self.region, - parts['ip']), - port=parts['port'], - path='/services/Cloud', - **kwargs) - - def split_clc_url(self, clc_url): - """Splits a cloud controller endpoint url.""" - parts = httplib.urlsplit(clc_url) - is_secure = parts.scheme == 'https' - ip, port = parts.netloc.split(':') - return {'ip': ip, 'port': int(port), 'is_secure': is_secure} - - def get_users(self): - """Grabs the list of all users.""" - return self.apiconn.get_list('DescribeUsers', {}, [('item', UserInfo)]) - - def get_user(self, name): - """Grab a single user by name.""" - try: - return self.apiconn.get_object('DescribeUser', - {'Name': name}, - UserInfo) - except boto.exception.BotoServerError, e: - if e.status == 400 and e.error_code == 'NotFound': - return None - raise - - def has_user(self, username): - """Determine if user exists.""" - return self.get_user(username) != None - - def create_user(self, username): - """Creates a new user, returning the userinfo object with - access/secret.""" - return self.apiconn.get_object('RegisterUser', {'Name': username}, - UserInfo) - - def delete_user(self, username): - """Deletes a user.""" - return self.apiconn.get_object('DeregisterUser', {'Name': username}, - UserInfo) - - def get_roles(self, project_roles=True): - """Returns a list of available roles.""" - return self.apiconn.get_list('DescribeRoles', - {'ProjectRoles': project_roles}, - [('item', UserRole)]) - - def get_user_roles(self, user, project=None): - """Returns a list of roles for the given user. - - Omitting project will return any global roles that the user has. - Specifying project will return only project specific roles. - - """ - params = {'User': user} - if project: - params['Project'] = project - return self.apiconn.get_list('DescribeUserRoles', - params, - [('item', UserRole)]) - - def add_user_role(self, user, role, project=None): - """Add a role to a user either globally or for a specific project.""" - return self.modify_user_role(user, role, project=project, - operation='add') - - def remove_user_role(self, user, role, project=None): - """Remove a role from a user either globally or for a specific - project.""" - return self.modify_user_role(user, role, project=project, - operation='remove') - - def modify_user_role(self, user, role, project=None, operation='add', - **kwargs): - """Add or remove a role for a user and project.""" - params = {'User': user, - 'Role': role, - 'Project': project, - 'Operation': operation} - return self.apiconn.get_status('ModifyUserRole', params) - - def get_projects(self, user=None): - """Returns a list of all projects.""" - if user: - params = {'User': user} - else: - params = {} - return self.apiconn.get_list('DescribeProjects', - params, - [('item', ProjectInfo)]) - - def get_project(self, name): - """Returns a single project with the specified name.""" - project = self.apiconn.get_object('DescribeProject', - {'Name': name}, - ProjectInfo) - - if project.projectname != None: - return project - - def create_project(self, projectname, manager_user, description=None, - member_users=None): - """Creates a new project.""" - params = {'Name': projectname, - 'ManagerUser': manager_user, - 'Description': description, - 'MemberUsers': member_users} - return self.apiconn.get_object('RegisterProject', params, ProjectInfo) - - def modify_project(self, projectname, manager_user=None, description=None): - """Modifies an existing project.""" - params = {'Name': projectname, - 'ManagerUser': manager_user, - 'Description': description} - return self.apiconn.get_status('ModifyProject', params) - - def delete_project(self, projectname): - """Permanently deletes the specified project.""" - return self.apiconn.get_object('DeregisterProject', - {'Name': projectname}, - ProjectInfo) - - def get_project_members(self, name): - """Returns a list of members of a project.""" - return self.apiconn.get_list('DescribeProjectMembers', - {'Name': name}, - [('item', ProjectMember)]) - - def add_project_member(self, user, project): - """Adds a user to a project.""" - return self.modify_project_member(user, project, operation='add') - - def remove_project_member(self, user, project): - """Removes a user from a project.""" - return self.modify_project_member(user, project, operation='remove') - - def modify_project_member(self, user, project, operation='add'): - """Adds or removes a user from a project.""" - params = {'User': user, - 'Project': project, - 'Operation': operation} - return self.apiconn.get_status('ModifyProjectMember', params) - - def get_zip(self, user, project): - """Returns the content of a zip file containing novarc and access - credentials.""" - params = {'Name': user, 'Project': project} - zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo) - return zip.file - - def get_hosts(self): - return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) - - def get_instance_types(self): - """Grabs the list of all users.""" - return self.apiconn.get_list('DescribeInstanceTypes', {}, - [('item', InstanceType)]) - - def block_ips(self, cidr): - """Block incoming traffic from specified hosts.""" - return self.apiconn.get_object('BlockExternalAddresses', - {'Cidr': cidr}, SimpleResponse) From 55dc316e5a3cf51307ebbac36b1b00e8c0d6ec70 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 5 Apr 2011 20:03:29 -0400 Subject: [PATCH 006/107] Testing for iptables manager changes. --- nova/tests/test_network.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 77f6aaff3..a67c846dd 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -164,3 +164,33 @@ class IptablesManagerTestCase(test.TestCase): self.assertTrue('-A %s -j run_tests.py-%s' \ % (chain, chain) in new_lines, "Built-in chain %s not wrapped" % (chain,)) + + def test_will_empty_chain(self): + self.manager.ipv4['filter'].add_chain('test-chain') + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP') + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain') + self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules)) + + def test_will_empty_unwrapped_chain(self): + self.manager.ipv4['filter'].add_chain('test-chain', wrap=False) + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP', + wrap=False) + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False) + self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules)) + + def test_will_not_empty_wrapped_when_unwrapped(self): + self.manager.ipv4['filter'].add_chain('test-chain') + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP') + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False) + self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules)) + + def test_will_not_empty_unwrapped_when_wrapped(self): + self.manager.ipv4['filter'].add_chain('test-chain', wrap=False) + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP', + wrap=False) + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain') + self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules)) From f86ea2bd81cc6fbdd0b1f4aeaae80645429f2b92 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 7 Apr 2011 01:42:49 -0400 Subject: [PATCH 007/107] test provider fw rules at the virt/ipteables layer. lowercase protocol names in admin api to match what the firewall driver expects. add provider fw rule chain in iptables6 as well. fix a couple of small typos and copy-paste errors. --- nova/tests/test_virt.py | 65 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 958c8e3e2..34ecd09c6 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -566,7 +566,9 @@ class IptablesFirewallTestCase(test.TestCase): self.network = utils.import_object(FLAGS.network_manager) class FakeLibvirtConnection(object): - pass + def nwfilterDefineXML(*args, **kwargs): + """setup_basic_rules in nwfilter calls this.""" + pass self.fake_libvirt_connection = FakeLibvirtConnection() self.fw = libvirt_conn.IptablesFirewallDriver( get_connection=lambda: self.fake_libvirt_connection) @@ -728,6 +730,67 @@ class IptablesFirewallTestCase(test.TestCase): "TCP port 80/81 acceptance rule wasn't added") db.instance_destroy(admin_ctxt, instance_ref['id']) + def test_provider_firewall_rules(self): + # keep from changing state of actual firewall + #def fake_function(*args, **kwargs): + # pass + #self.fw.iptables.apply = fake_function + #self.fw.nwfilter.setup_basic_filtering = fake_function + + # setup basic instance data + instance_ref = db.instance_create(self.context, + {'user_id': 'fake', + 'project_id': 'fake', + 'mac_address': '56:12:12:12:12:12'}) + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + admin_ctxt = context.get_admin_context() + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': instance_ref['id']}) + # FRAGILE: peeks at how the firewall names chains + chain_name = 'inst-%s' % instance_ref['id'] + + # create a firewall via setup_basic_filtering like libvirt_conn.spawn + # should have a chain with 0 rules + self.fw.setup_basic_filtering(instance_ref, network_info=None) + self.assertTrue('provider' in self.fw.iptables.ipv4['filter'].chains) + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(0, len(rules)) + + # add a rule and send the update message, check for 1 rule + provider_fw0 = db.provider_fw_rule_create(admin_ctxt, + {'protocol': 'tcp', + 'cidr': '10.99.99.99/32', + 'from_port': 1, + 'to_port': 65535}) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(1, len(rules)) + + # Add another, refresh, and make sure number of rules goes to two + provider_fw1 = db.provider_fw_rule_create(admin_ctxt, + {'protocol': 'udp', + 'cidr': '10.99.99.99/32', + 'from_port': 1, + 'to_port': 65535}) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(2, len(rules)) + + # create the instance filter and make sure it has a jump rule + self.fw.prepare_instance_filter(instance_ref, network_info=None) + self.fw.apply_instance_filter(instance_ref) + inst_rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == chain_name] + jump_rules = [rule for rule in inst_rules if '-j' in rule.rule] + prov_rules = [rule for rule in jump_rules if 'provider' in rule.rule] + self.assertEqual(1, len(prov_rules)) + class NWFilterTestCase(test.TestCase): def setUp(self): From 8f4dc30f7759d928857a11534124505b85eb86b2 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 7 Apr 2011 13:22:03 -0400 Subject: [PATCH 008/107] remove unused code. --- nova/tests/test_virt.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 34ecd09c6..7c3dbf654 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -731,12 +731,6 @@ class IptablesFirewallTestCase(test.TestCase): db.instance_destroy(admin_ctxt, instance_ref['id']) def test_provider_firewall_rules(self): - # keep from changing state of actual firewall - #def fake_function(*args, **kwargs): - # pass - #self.fw.iptables.apply = fake_function - #self.fw.nwfilter.setup_basic_filtering = fake_function - # setup basic instance data instance_ref = db.instance_create(self.context, {'user_id': 'fake', From b8cd7449fe25d0052928e46077391115db72d05d Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Wed, 25 May 2011 15:48:03 -0400 Subject: [PATCH 009/107] Initial tests --- nova/tests/test_instance_types_metadata.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 nova/tests/test_instance_types_metadata.py diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py new file mode 100644 index 000000000..a22bf2b77 --- /dev/null +++ b/nova/tests/test_instance_types_metadata.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 University of Southern California +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Unit Tests for instance types metadata code +""" + + +from nova import test +from nova.db.sqlalchemy.session import get_session +from nova.db.sqlalchemy import models + + +class InstanceTypeMetadataTestCase(test.TestCase): + + def setUp(self): + super(InstanceTypeMetadataTestCase, self).setUp() + values = dict(memory_mb=22000, + vcpus=8, + local_gb=1690, + flavorid=105) + metadata = dict(cpu_arch="x86_64", + cpu_info=dict(model="Nehalem"), + xpu_arch="fermi", + xpus=2, + xpu_info=dict(model="Tesla 2050", gcores="448"), + net_arch="ethernet", + net_mbps=10000) + + metadata_refs = [] + for k,v in metadata.iteritems(): + metadata_ref = models.InstanceTypeMetadata() + metadata_ref['key'] = k + metadata_ref['value'] = v + metadata_refs.append(metadata_ref) + values['metadata'] = metadata_refs + + instance_type_ref = models.InstanceTypes() + instance_type_ref.update(values) + + session = get_session() + with session.begin(): + instance_type_ref.save(session=session) + # Add cg1.4xlarge + + + def test_foo(self): + # Add a new instance type cg1 + # Add the metadata + # Retrieve the metadata + pass From 4fa0fa518ff3f627bcf7ffa603459ba13d989233 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 26 May 2011 12:09:04 -0400 Subject: [PATCH 010/107] Make a cleaner log message and use [] instead of . to get database fields. --- nova/tests/test_libvirt.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 1743b09a2..0eaf069fb 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -884,10 +884,7 @@ class IptablesFirewallTestCase(test.TestCase): def test_provider_firewall_rules(self): # setup basic instance data - instance_ref = db.instance_create(self.context, - {'user_id': 'fake', - 'project_id': 'fake', - 'mac_address': '56:12:12:12:12:12'}) + instance_ref = self._create_instance_ref() ip = '10.11.12.13' network_ref = db.project_get_network(self.context, 'fake') admin_ctxt = context.get_admin_context() From 70bee085928895f57adb4aa0c571be0c42bcfb06 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 26 May 2011 12:48:07 -0400 Subject: [PATCH 011/107] Test tweaks. --- nova/tests/test_libvirt.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 0eaf069fb..bb7c8cf33 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -885,6 +885,7 @@ class IptablesFirewallTestCase(test.TestCase): def test_provider_firewall_rules(self): # setup basic instance data instance_ref = self._create_instance_ref() + nw_info = _create_network_info(1) ip = '10.11.12.13' network_ref = db.project_get_network(self.context, 'fake') admin_ctxt = context.get_admin_context() @@ -897,7 +898,7 @@ class IptablesFirewallTestCase(test.TestCase): # create a firewall via setup_basic_filtering like libvirt_conn.spawn # should have a chain with 0 rules - self.fw.setup_basic_filtering(instance_ref, network_info=None) + self.fw.setup_basic_filtering(instance_ref, network_info=nw_info) self.assertTrue('provider' in self.fw.iptables.ipv4['filter'].chains) rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules if rule.chain == 'provider'] @@ -926,13 +927,17 @@ class IptablesFirewallTestCase(test.TestCase): self.assertEqual(2, len(rules)) # create the instance filter and make sure it has a jump rule - self.fw.prepare_instance_filter(instance_ref, network_info=None) + self.fw.prepare_instance_filter(instance_ref, network_info=nw_info) self.fw.apply_instance_filter(instance_ref) inst_rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules if rule.chain == chain_name] jump_rules = [rule for rule in inst_rules if '-j' in rule.rule] - prov_rules = [rule for rule in jump_rules if 'provider' in rule.rule] - self.assertEqual(1, len(prov_rules)) + provjump_rules = [] + # IptablesTable doesn't make rules unique internally + for rule in jump_rules: + if 'provider' in rule.rule and rule not in provjump_rules: + provjump_rules.append(rule) + self.assertEqual(1, len(provjump_rules)) class NWFilterTestCase(test.TestCase): From b7683abcc30dfa09b88bcffcabc1e32dd8f479a1 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:03:36 -0400 Subject: [PATCH 012/107] Adding accessor methods for instance type metadata --- nova/exception.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nova/exception.py b/nova/exception.py index 56c20d111..4e5f750c1 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -474,6 +474,11 @@ class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") +class InstanceTypeMetadataNotFound(NotFound): + message = _("Instance Type %(instance_type_id)s has no metadata with " + "key %(metadata_key)s.") + + class LDAPObjectNotFound(NotFound): message = _("LDAP object could not be found") From 4de43f8d63098b3e10de73f99765325004adb325 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:10:19 -0400 Subject: [PATCH 013/107] Adding test code --- nova/tests/test_instance_types_metadata.py | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index a22bf2b77..06c11a7cf 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -16,7 +16,9 @@ Unit Tests for instance types metadata code """ +import nova.db.api +from nova import context from nova import test from nova.db.sqlalchemy.session import get_session from nova.db.sqlalchemy import models @@ -26,15 +28,16 @@ class InstanceTypeMetadataTestCase(test.TestCase): def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() - values = dict(memory_mb=22000, + values = dict(name="cg1.4xlarge", + memory_mb=22000, vcpus=8, local_gb=1690, flavorid=105) metadata = dict(cpu_arch="x86_64", - cpu_info=dict(model="Nehalem"), + cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_info=dict(model="Tesla 2050", gcores="448"), + xpu_model="Tesla 2050", net_arch="ethernet", net_mbps=10000) @@ -44,19 +47,22 @@ class InstanceTypeMetadataTestCase(test.TestCase): metadata_ref['key'] = k metadata_ref['value'] = v metadata_refs.append(metadata_ref) - values['metadata'] = metadata_refs + values['meta'] = metadata_refs instance_type_ref = models.InstanceTypes() instance_type_ref.update(values) + session = get_session() with session.begin(): instance_type_ref.save(session=session) - # Add cg1.4xlarge + self.instance_type_id = instance_type_ref.id + def test_instance_type_metadata_get(self): + self.assertEquals( \ + nova.db.api.instance_type_metadata_get(context.get_admin_context(), + self.instance_type_id), + {'foo' : 'bar'}) - def test_foo(self): - # Add a new instance type cg1 - # Add the metadata - # Retrieve the metadata - pass + + \ No newline at end of file From ff2caf43a9858e853061316e45c3f1a516990de9 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:29:51 -0400 Subject: [PATCH 014/107] Added a unit test --- nova/tests/test_instance_types_metadata.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 06c11a7cf..5d8364c3a 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -16,9 +16,8 @@ Unit Tests for instance types metadata code """ -import nova.db.api - from nova import context +from nova import db from nova import test from nova.db.sqlalchemy.session import get_session from nova.db.sqlalchemy import models @@ -59,10 +58,17 @@ class InstanceTypeMetadataTestCase(test.TestCase): self.instance_type_id = instance_type_ref.id def test_instance_type_metadata_get(self): - self.assertEquals( \ - nova.db.api.instance_type_metadata_get(context.get_admin_context(), - self.instance_type_id), - {'foo' : 'bar'}) + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + retrieved_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, retrieved_metadata) \ No newline at end of file From e4c1a59195288c1b7e88a705a99394de67b79a06 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:39:37 -0400 Subject: [PATCH 015/107] Added delete instance metadata unit test --- nova/tests/test_instance_types_metadata.py | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 5d8364c3a..c83c5bfca 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -47,11 +47,8 @@ class InstanceTypeMetadataTestCase(test.TestCase): metadata_ref['value'] = v metadata_refs.append(metadata_ref) values['meta'] = metadata_refs - instance_type_ref = models.InstanceTypes() instance_type_ref.update(values) - - session = get_session() with session.begin(): instance_type_ref.save(session=session) @@ -65,10 +62,22 @@ class InstanceTypeMetadataTestCase(test.TestCase): xpu_model="Tesla 2050", net_arch="ethernet", net_mbps="10000") - retrieved_metadata = db.api.instance_type_metadata_get( + actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) - self.assertEquals(expected_metadata, retrieved_metadata) - - - \ No newline at end of file + self.assertEquals(expected_metadata, actual_metadata) + + def test_instance_type_metadata_delete(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_delete(context.get_admin_context(), + self.instance_type_id, + "xpu_model") + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) \ No newline at end of file From 65ad5225363ddb1b224d290df3786564685ac801 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:44:40 -0400 Subject: [PATCH 016/107] Added test for instance type metadata update --- nova/tests/test_instance_types_metadata.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index c83c5bfca..d72a72e0d 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -80,4 +80,24 @@ class InstanceTypeMetadataTestCase(test.TestCase): actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) \ No newline at end of file + self.assertEquals(expected_metadata, actual_metadata) + + def test_instance_type_metadata_update(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Sandy Bridge", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_update_or_create( + context.get_admin_context(), + self.instance_type_id, + dict(cpu_model="Sandy Bridge")) + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) + + + \ No newline at end of file From b0f42e2a9fc61bbb908ef32f300ded319d4c8f43 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:54:19 -0400 Subject: [PATCH 017/107] Added test for instance type metadata create --- nova/tests/test_instance_types_metadata.py | 44 ++++++++++++++-------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index d72a72e0d..085a951a1 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -36,10 +36,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps=10000) - + xpu_model="Tesla 2050") metadata_refs = [] for k,v in metadata.iteritems(): metadata_ref = models.InstanceTypeMetadata() @@ -54,14 +51,18 @@ class InstanceTypeMetadataTestCase(test.TestCase): instance_type_ref.save(session=session) self.instance_type_id = instance_type_ref.id + def tearDown(self): + # Remove the instance from the database + db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") + super(InstanceTypeMetadataTestCase, self).tearDown() + + def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") + xpu_model="Tesla 2050") actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) @@ -71,9 +72,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", - xpus="2", - net_arch="ethernet", - net_mbps="10000") + xpus="2") db.api.instance_type_metadata_delete(context.get_admin_context(), self.instance_type_id, "xpu_model") @@ -87,9 +86,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Sandy Bridge", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") + xpu_model="Tesla 2050") db.api.instance_type_metadata_update_or_create( context.get_admin_context(), self.instance_type_id, @@ -98,6 +95,21 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - - - \ No newline at end of file + + def test_instance_type_metadata_create(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_update_or_create( + context.get_admin_context(), + self.instance_type_id, + dict(net_arch="ethernet", + net_mbps=10000)) + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) From 16978f5c1579d5175978bedeb2324231c3cc390f Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 14:20:08 -0400 Subject: [PATCH 018/107] Modified instance_type_create to take metadata --- nova/tests/test_instance_types_metadata.py | 23 +++++++--------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 085a951a1..5263b1cba 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -27,6 +27,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() + self.context = context.get_admin_context() values = dict(name="cg1.4xlarge", memory_mb=22000, vcpus=8, @@ -36,27 +37,17 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_model="Tesla 2050") - metadata_refs = [] - for k,v in metadata.iteritems(): - metadata_ref = models.InstanceTypeMetadata() - metadata_ref['key'] = k - metadata_ref['value'] = v - metadata_refs.append(metadata_ref) - values['meta'] = metadata_refs - instance_type_ref = models.InstanceTypes() - instance_type_ref.update(values) - session = get_session() - with session.begin(): - instance_type_ref.save(session=session) - self.instance_type_id = instance_type_ref.id + xpu_model="Tesla 2050") + values['meta'] = metadata + ref = db.api.instance_type_create(self.context, + values) + self.instance_type_id = ref.id def tearDown(self): - # Remove the instance from the database + # Remove the instance type from the database db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") super(InstanceTypeMetadataTestCase, self).tearDown() - def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", From b67b3ea9d7d376dddeeb0e7f307dc2130cca69d3 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 14:29:23 -0400 Subject: [PATCH 019/107] Fixing pep8 problems --- nova/exception.py | 2 +- nova/tests/test_instance_types_metadata.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 4e5f750c1..a08311e19 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -474,12 +474,12 @@ class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") + class InstanceTypeMetadataNotFound(NotFound): message = _("Instance Type %(instance_type_id)s has no metadata with " "key %(metadata_key)s.") - class LDAPObjectNotFound(NotFound): message = _("LDAP object could not be found") diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 5263b1cba..1c4185888 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -24,7 +24,7 @@ from nova.db.sqlalchemy import models class InstanceTypeMetadataTestCase(test.TestCase): - + def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() self.context = context.get_admin_context() @@ -42,12 +42,12 @@ class InstanceTypeMetadataTestCase(test.TestCase): ref = db.api.instance_type_create(self.context, values) self.instance_type_id = ref.id - + def tearDown(self): # Remove the instance type from the database db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") super(InstanceTypeMetadataTestCase, self).tearDown() - + def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", @@ -58,20 +58,20 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_delete(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2") - db.api.instance_type_metadata_delete(context.get_admin_context(), + db.api.instance_type_metadata_delete(context.get_admin_context(), self.instance_type_id, "xpu_model") actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_update(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Sandy Bridge", @@ -86,13 +86,13 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_create(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", + xpu_model="Tesla 2050", net_arch="ethernet", net_mbps="10000") db.api.instance_type_metadata_update_or_create( From 70f95d5e2b663ebc423e69bbd8e695d7116219a8 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Mon, 6 Jun 2011 15:32:48 -0400 Subject: [PATCH 020/107] Remove ipy from nova-manage and use netaddr --- bin/nova-manage | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b0cd343f5..5ac0b8a0c 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -56,11 +56,11 @@ import gettext import glob import json +import netaddr import os import sys import time -import IPy # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -513,7 +513,7 @@ class FloatingIpCommands(object): def create(self, host, range): """Creates floating ips for host by range arguments: host ip_range""" - for address in IPy.IP(range): + for address in netaddr.IPRange(range): db.floating_ip_create(context.get_admin_context(), {'address': str(address), 'host': host}) @@ -521,7 +521,7 @@ class FloatingIpCommands(object): def delete(self, ip_range): """Deletes floating ips by range arguments: range""" - for address in IPy.IP(ip_range): + for address in netaddr.IPRange(ip_range): db.floating_ip_destroy(context.get_admin_context(), str(address)) From 1edf25c2a4e9ca3bf518e9afd2fe98ff44767380 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Mon, 6 Jun 2011 15:34:51 -0400 Subject: [PATCH 021/107] Remove ipy from network code and replace with netaddr --- nova/tests/test_flat_network.py | 6 +++--- nova/tests/test_vlan_network.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/tests/test_flat_network.py b/nova/tests/test_flat_network.py index dcc617e25..8544019c0 100644 --- a/nova/tests/test_flat_network.py +++ b/nova/tests/test_flat_network.py @@ -18,7 +18,7 @@ """ Unit Tests for flat network code """ -import IPy +import netaddr import os import unittest @@ -45,8 +45,8 @@ class FlatNetworkTestCase(base.NetworkTestCase): self.context._project = self.projects[0] self.context.project_id = self.projects[0].id - pubnet = IPy.IP(flags.FLAGS.floating_range) - address = str(pubnet[0]) + pubnet = netaddr.IPRange(flags.FLAGS.floating_range) + address = str(list(pubnet)[0]) try: db.floating_ip_get_by_address(context.get_admin_context(), address) except exception.NotFound: diff --git a/nova/tests/test_vlan_network.py b/nova/tests/test_vlan_network.py index 063b81832..a1c8ab11c 100644 --- a/nova/tests/test_vlan_network.py +++ b/nova/tests/test_vlan_network.py @@ -18,7 +18,7 @@ """ Unit Tests for vlan network code """ -import IPy +import netaddr import os from nova import context @@ -44,8 +44,8 @@ class VlanNetworkTestCase(base.NetworkTestCase): # TODO(vish): better way of adding floating ips self.context._project = self.projects[0] self.context.project_id = self.projects[0].id - pubnet = IPy.IP(flags.FLAGS.floating_range) - address = str(pubnet[0]) + pubnet = netaddr.IPNetwork(flags.FLAGS.floating_range) + address = str(list(pubnet)[0]) try: db.floating_ip_get_by_address(context.get_admin_context(), address) except exception.NotFound: From 19d3181b673c474732039d7fa4a5b76e28069755 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Mon, 6 Jun 2011 17:03:50 -0400 Subject: [PATCH 022/107] Convert stray import IPy --- nova/tests/test_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 77f6aaff3..da92f2066 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -18,7 +18,7 @@ """ Unit Tests for network code """ -import IPy +import netaddr import os from nova import test From 8e2f9306b8a62602710eb69265809cf7d5c38c1e Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Tue, 7 Jun 2011 10:39:30 -0400 Subject: [PATCH 023/107] Use IPNetwork rather than IPRange --- bin/nova-manage | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 5ac0b8a0c..1f8ccf268 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -513,7 +513,7 @@ class FloatingIpCommands(object): def create(self, host, range): """Creates floating ips for host by range arguments: host ip_range""" - for address in netaddr.IPRange(range): + for address in netaddr.IPNetwork(range): db.floating_ip_create(context.get_admin_context(), {'address': str(address), 'host': host}) @@ -521,7 +521,7 @@ class FloatingIpCommands(object): def delete(self, ip_range): """Deletes floating ips by range arguments: range""" - for address in netaddr.IPRange(ip_range): + for address in netaddr.IPNetwork(ip_range): db.floating_ip_destroy(context.get_admin_context(), str(address)) From 1d4ce45f1db8ee1820f83179f5456e4b26f86c5e Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Sat, 11 Jun 2011 15:14:46 -0400 Subject: [PATCH 024/107] pep 8 whitespace fix. --- nova/tests/test_libvirt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 07c8e81f1..ee94d3c17 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -1058,7 +1058,6 @@ class IptablesFirewallTestCase(test.TestCase): db.instance_destroy(admin_ctxt, instance_ref['id']) - def test_provider_firewall_rules(self): # setup basic instance data instance_ref = self._create_instance_ref() From a09094236aed84028207f4a95b1427d1d2ffb918 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Thu, 16 Jun 2011 16:33:29 -0400 Subject: [PATCH 026/107] Rename: intance_type_metadata -> instance_type_extra_specs --- nova/exception.py | 6 +- nova/tests/test_instance_types_metadata.py | 106 --------------------- 2 files changed, 3 insertions(+), 109 deletions(-) delete mode 100644 nova/tests/test_instance_types_metadata.py diff --git a/nova/exception.py b/nova/exception.py index ce287b7d6..394004fe6 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -482,9 +482,9 @@ class InstanceMetadataNotFound(NotFound): "key %(metadata_key)s.") -class InstanceTypeMetadataNotFound(NotFound): - message = _("Instance Type %(instance_type_id)s has no metadata with " - "key %(metadata_key)s.") +class InstanceTypeExtraSpecsNotFound(NotFound): + message = _("Instance Type %(instance_type_id)s has no extra specs with " + "key %(extra_specs_key)s.") class LDAPObjectNotFound(NotFound): diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py deleted file mode 100644 index 1c4185888..000000000 --- a/nova/tests/test_instance_types_metadata.py +++ /dev/null @@ -1,106 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 University of Southern California -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Unit Tests for instance types metadata code -""" - -from nova import context -from nova import db -from nova import test -from nova.db.sqlalchemy.session import get_session -from nova.db.sqlalchemy import models - - -class InstanceTypeMetadataTestCase(test.TestCase): - - def setUp(self): - super(InstanceTypeMetadataTestCase, self).setUp() - self.context = context.get_admin_context() - values = dict(name="cg1.4xlarge", - memory_mb=22000, - vcpus=8, - local_gb=1690, - flavorid=105) - metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus=2, - xpu_model="Tesla 2050") - values['meta'] = metadata - ref = db.api.instance_type_create(self.context, - values) - self.instance_type_id = ref.id - - def tearDown(self): - # Remove the instance type from the database - db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") - super(InstanceTypeMetadataTestCase, self).tearDown() - - def test_instance_type_metadata_get(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050") - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_delete(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2") - db.api.instance_type_metadata_delete(context.get_admin_context(), - self.instance_type_id, - "xpu_model") - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_update(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Sandy Bridge", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050") - db.api.instance_type_metadata_update_or_create( - context.get_admin_context(), - self.instance_type_id, - dict(cpu_model="Sandy Bridge")) - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_create(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") - db.api.instance_type_metadata_update_or_create( - context.get_admin_context(), - self.instance_type_id, - dict(net_arch="ethernet", - net_mbps=10000)) - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) From 0d8cb633b71a43f7909fe4ecaf56bc05080430f7 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sat, 18 Jun 2011 23:10:41 -0400 Subject: [PATCH 027/107] General cleanup and refactor of a lot of the API/WSGI service code. --- bin/nova-api | 18 ++++++++-- nova/log.py | 13 +++++++ nova/service.py | 90 ++++++++++++++++++------------------------------- nova/test.py | 23 ------------- nova/wsgi.py | 61 ++++++++++++++++----------------- 5 files changed, 92 insertions(+), 113 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a1088c23d..6db68be9c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,6 +24,8 @@ import gettext import os import sys +import eventlet.pool + # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -46,6 +48,13 @@ LOG = logging.getLogger('nova.api') FLAGS = flags.FLAGS + +def launch(service_name): + _service = service.WSGIService(service_name) + _service.start() + _service.wait() + + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) @@ -57,5 +66,10 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - service = service.serve_wsgi(service.ApiService) - service.wait() + + pool = eventlet.pool.Pool() + pool.execute(launch, "ec2") + pool.execute(launch, "osapi") + pool.wait_all() + + print >>sys.stderr, "Exiting..." diff --git a/nova/log.py b/nova/log.py index 6909916a1..d3bffdb21 100644 --- a/nova/log.py +++ b/nova/log.py @@ -314,3 +314,16 @@ logging.setLoggerClass(NovaLogger) def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" logging.root.log(AUDIT, msg, *args, **kwargs) + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.DEBUG): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + diff --git a/nova/service.py b/nova/service.py index 74f9f04d8..768390414 100644 --- a/nova/service.py +++ b/nova/service.py @@ -232,45 +232,45 @@ class Service(object): logging.exception(_('model server went away')) -class WsgiService(object): - """Base class for WSGI based services. +class WSGIService(object): + """Provides ability to launch API from a 'paste' configuration.""" - For each api you define, you must also define these flags: - :_listen: The address on which to listen - :_listen_port: The port on which to listen + def __init__(self, name, config_name=None): + """Initialize, but do not start, an API service.""" + self.name = name + self._config_name = config_name or FLAGS.api_paste_config + self._config_location = self._find_config() + self._config = self._load_config() + self.application = self._load_application() + host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + port = getattr(FLAGS, '%s_listen_port' % name, 0) + self.server = wsgi.Server(name, self.application, host, port) - """ + def _find_config(self): + """Attempt to find 'paste' configuration file.""" + location = wsgi.paste_config_file(self._config_name) + logging.debug(_("Using paste.deploy config at: %s"), location) + return location - def __init__(self, conf, apis): - self.conf = conf - self.apis = apis - self.wsgi_app = None + def _load_config(self): + """Read and return the 'paste' configuration file.""" + return wsgi.load_paste_configuration(self._config_location, self.name) + + def _load_application(self): + """Using the loaded configuration, return the WSGI application.""" + return wsgi.load_paste_app(self._config_location, self.name) def start(self): - self.wsgi_app = _run_wsgi(self.conf, self.apis) + """Start serving this API using loaded configuration.""" + self.server.start() + + def stop(self): + """Stop serving this API.""" + self.server.stop() def wait(self): - self.wsgi_app.wait() - - def get_socket_info(self, api_name): - """Returns the (host, port) that an API was started on.""" - return self.wsgi_app.socket_info[api_name] - - -class ApiService(WsgiService): - """Class for our nova-api service.""" - - @classmethod - def create(cls, conf=None): - if not conf: - conf = wsgi.paste_config_file(FLAGS.api_paste_config) - if not conf: - message = (_('No paste configuration found for: %s'), - FLAGS.api_paste_config) - raise exception.Error(message) - api_endpoints = ['ec2', 'osapi'] - service = cls(conf, api_endpoints) - return service + """Wait for the service to stop serving this API.""" + self.server.wait() def serve(*services): @@ -321,29 +321,3 @@ def serve_wsgi(cls, conf=None): service.start() return service - - -def _run_wsgi(paste_config_file, apis): - logging.debug(_('Using paste.deploy config at: %s'), paste_config_file) - apps = [] - for api in apis: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - logging.debug(_('No paste configuration for app: %s'), api) - continue - logging.debug(_('App Config: %(api)s\n%(config)r') % locals()) - logging.info(_('Running %s API'), api) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, - getattr(FLAGS, '%s_listen_port' % api), - getattr(FLAGS, '%s_listen' % api), - api)) - if len(apps) == 0: - logging.error(_('No known API applications configured in %s.'), - paste_config_file) - return - - server = wsgi.Server() - for app in apps: - server.start(*app) - return server diff --git a/nova/test.py b/nova/test.py index 4a0a18fe7..ab1eaf5fd 100644 --- a/nova/test.py +++ b/nova/test.py @@ -38,7 +38,6 @@ from nova import flags from nova import rpc from nova import utils from nova import service -from nova import wsgi from nova.virt import fake @@ -81,7 +80,6 @@ class TestCase(unittest.TestCase): self.injected = [] self._services = [] self._monkey_patch_attach() - self._monkey_patch_wsgi() self._original_flags = FLAGS.FlagValuesDict() rpc.ConnectionPool = rpc.Pool(max_size=FLAGS.rpc_conn_pool_size) @@ -107,7 +105,6 @@ class TestCase(unittest.TestCase): # Reset our monkey-patches rpc.Consumer.attach_to_eventlet = self.original_attach - wsgi.Server.start = self.original_start # Stop any timers for x in self.injected: @@ -163,26 +160,6 @@ class TestCase(unittest.TestCase): _wrapped.func_name = self.original_attach.func_name rpc.Consumer.attach_to_eventlet = _wrapped - def _monkey_patch_wsgi(self): - """Allow us to kill servers spawned by wsgi.Server.""" - self.original_start = wsgi.Server.start - - @functools.wraps(self.original_start) - def _wrapped_start(inner_self, *args, **kwargs): - original_spawn_n = inner_self.pool.spawn_n - - @functools.wraps(original_spawn_n) - def _wrapped_spawn_n(*args, **kwargs): - rv = greenthread.spawn(*args, **kwargs) - self._services.append(rv) - - inner_self.pool.spawn_n = _wrapped_spawn_n - self.original_start(inner_self, *args, **kwargs) - inner_self.pool.spawn_n = original_spawn_n - - _wrapped_start.func_name = self.original_start.func_name - wsgi.Server.start = _wrapped_start - # Useful assertions def assertDictMatch(self, d1, d2, approx_equal=False, tolerance=0.001): """Assert two dicts are equivalent. diff --git a/nova/wsgi.py b/nova/wsgi.py index 33ba852bc..439ec6a12 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -43,46 +43,45 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.wsgi') -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.DEBUG): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg) - - class Server(object): """Server class to manage multiple WSGI sockets and applications.""" - def __init__(self, threads=1000): - self.pool = eventlet.GreenPool(threads) - self.socket_info = {} + default_pool_size = 1000 + logger_name = "eventlet.wsgi.server" - def start(self, application, port, host='0.0.0.0', key=None, backlog=128): - """Run a WSGI server with the given application.""" - arg0 = sys.argv[0] - logging.audit(_('Starting %(arg0)s on %(host)s:%(port)s') % locals()) - socket = eventlet.listen((host, port), backlog=backlog) - self.pool.spawn_n(self._run, application, socket) - if key: - self.socket_info[key] = socket.getsockname() + def __init__(self, name, app, host, port, pool_size=None): + self.name = name + self.app = app + self.host = host + self.port = port + self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) + self._log = logging.WritableLogger(logging.getLogger(self.logger_name)) + + def _start(self, socket): + """Blocking eventlet WSGI server launched from the real 'start'.""" + eventlet.wsgi.server(socket, + self.app, + custom_pool=self._pool, + log=self._log) + + def start(self, backlog=128): + """Serve given WSGI application using the given parameters.""" + socket = eventlet.listen((self.host, self.port), backlog=backlog) + self._server = eventlet.spawn(self._start, socket) + (self.host, self.port) = socket.getsockname() + LOG.info(_('Starting %(app)s on %(host)s:%(port)s') % self.__dict__) + + def stop(self): + """Stop this server by killing the greenthread running it.""" + self._server.kill() def wait(self): - """Wait until all servers have completed running.""" + """Wait until server has been stopped.""" try: - self.pool.waitall() + self._server.wait() except KeyboardInterrupt: pass - def _run(self, application, socket): - """Start a WSGI server in a new green thread.""" - logger = logging.getLogger('eventlet.wsgi.server') - eventlet.wsgi.server(socket, application, custom_pool=self.pool, - log=WritableLogger(logger)) - class Request(webob.Request): pass @@ -340,6 +339,8 @@ def paste_config_file(basename): if os.path.exists(configfile): return configfile + raise Exception(_("Unable to find paste.deploy config '%s'") % basename) + def load_paste_configuration(filename, appname): """Returns a paste configuration dict, or None.""" From 658c7d955a42c46d9f6b18a6290c9a78c4160980 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:09:09 -0400 Subject: [PATCH 028/107] Removed debugging, made objectstore tests pass again. --- bin/nova-api | 3 --- nova/log.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 6db68be9c..90c8b69ad 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -66,10 +66,7 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() - - print >>sys.stderr, "Exiting..." diff --git a/nova/log.py b/nova/log.py index d3bffdb21..3080daee2 100644 --- a/nova/log.py +++ b/nova/log.py @@ -325,5 +325,3 @@ class WritableLogger(object): def write(self, msg): self.logger.log(self.level, msg) - - From d699f2919f34d3a3d69e88d6430b75d5ab979255 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:41:42 -0400 Subject: [PATCH 029/107] Cleaned up nova-api binary and logging a bit. --- bin/nova-api | 32 +++++++++----------------------- nova/__init__.py | 12 ++++++++++++ nova/service.py | 19 ------------------- nova/utils.py | 2 ++ nova/wsgi.py | 2 +- setup.cfg | 3 +++ 6 files changed, 27 insertions(+), 43 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 90c8b69ad..7d80b0b78 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -20,33 +20,20 @@ """Starter script for Nova API.""" -import gettext -import os import sys import eventlet.pool -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - from nova import flags from nova import log as logging from nova import service from nova import utils from nova import version -from nova import wsgi -LOG = logging.getLogger('nova.api') - +LOG = logging.getLogger("nova.api") FLAGS = flags.FLAGS +VERSION = version.version_string_with_vcs() def launch(service_name): @@ -55,18 +42,17 @@ def launch(service_name): _service.wait() -if __name__ == '__main__': +def main(): utils.default_flagfile() FLAGS(sys.argv) - logging.setup() - LOG.audit(_("Starting nova-api node (version %s)"), - version.version_string_with_vcs()) - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) +# logging.setup() + LOG.audit(_("Starting nova-api node (version %s)") % VERSION) pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/nova/__init__.py b/nova/__init__.py index 256db55a9..7b05611b9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,3 +30,15 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ + +import gettext + +import log as logging + + +def initialize(): + gettext.install("nova", unicode=1) + logging.setup() + + +initialize() diff --git a/nova/service.py b/nova/service.py index 768390414..ec2de9004 100644 --- a/nova/service.py +++ b/nova/service.py @@ -302,22 +302,3 @@ def serve(*services): def wait(): while True: greenthread.sleep(5) - - -def serve_wsgi(cls, conf=None): - try: - service = cls.create(conf) - except Exception: - logging.exception('in WsgiService.create()') - raise - finally: - # After we've loaded up all our dynamic bits, check - # whether we should print help - flags.DEFINE_flag(flags.HelpFlag()) - flags.DEFINE_flag(flags.HelpshortFlag()) - flags.DEFINE_flag(flags.HelpXMLFlag()) - FLAGS.ParseNewFlags() - - service.start() - - return service diff --git a/nova/utils.py b/nova/utils.py index 691134ada..22d0ce940 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -238,6 +238,8 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' + if not os.path.exists(filename): + return flagfile = ['--flagfile=%s' % filename] sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] diff --git a/nova/wsgi.py b/nova/wsgi.py index 439ec6a12..4da8eff82 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -69,7 +69,7 @@ class Server(object): socket = eventlet.listen((self.host, self.port), backlog=backlog) self._server = eventlet.spawn(self._start, socket) (self.host, self.port) = socket.getsockname() - LOG.info(_('Starting %(app)s on %(host)s:%(port)s') % self.__dict__) + LOG.info(_('Starting %(name)s on %(host)s:%(port)s') % self.__dict__) def stop(self): """Stop this server by killing the greenthread running it.""" diff --git a/setup.cfg b/setup.cfg index 9c0a331e3..7efc46f23 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[develop] +user = True + [build_sphinx] all_files = 1 build-dir = doc/build From 7d3c46eea5540feb0b462e504ad4aeeb282e262e Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 16:27:46 -0400 Subject: [PATCH 030/107] Further nova-api cleanup. --- bin/nova-api | 65 ++++++++++++++++++++++++++++-------------------- nova/__init__.py | 1 + nova/log.py | 2 +- nova/service.py | 2 +- nova/wsgi.py | 5 +--- 5 files changed, 42 insertions(+), 33 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 7d80b0b78..ff4aa83d1 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,7 +1,5 @@ #!/usr/bin/env python -# pylint: disable=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - +# # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -18,40 +16,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Starter script for Nova API.""" +"""Starter script for Nova API. + +Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings +about re-imports should be ignored. + +""" + +# pylint: disable=W0404 import sys +import multiprocessing -import eventlet.pool - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import version - - -LOG = logging.getLogger("nova.api") -FLAGS = flags.FLAGS -VERSION = version.version_string_with_vcs() +import nova.flags +import nova.log +import nova.service +import nova.version +import nova.utils def launch(service_name): - _service = service.WSGIService(service_name) - _service.start() - _service.wait() + """Launch WSGI service with name matching 'paste' config file section.""" + service = nova.service.WSGIService(service_name) + service.start() + try: + service.wait() + except KeyboardInterrupt: + service.stop() def main(): - utils.default_flagfile() - FLAGS(sys.argv) -# logging.setup() - LOG.audit(_("Starting nova-api node (version %s)") % VERSION) + """Begin process of launching both EC2 and OSAPI services.""" + version = nova.version.version_string_with_vcs() + logger = nova.log.getLogger("nova.api") + logger.audit(_("Starting nova-api node (version %s)") % version) - pool = eventlet.pool.Pool() - pool.execute(launch, "ec2") - pool.execute(launch, "osapi") - pool.wait_all() + nova.flags.FLAGS(sys.argv) + nova.utils.default_flagfile() + + pool = multiprocessing.Pool(2) + pool.map_async(launch, ["ec2", "osapi"]) + pool.close() + + try: + pool.join() + except KeyboardInterrupt: + logger.audit(_("Exiting...")) + pool.terminate() if __name__ == '__main__': diff --git a/nova/__init__.py b/nova/__init__.py index 7b05611b9..886eaee20 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -39,6 +39,7 @@ import log as logging def initialize(): gettext.install("nova", unicode=1) logging.setup() + logging.debug(_("Initialized logging.")) initialize() diff --git a/nova/log.py b/nova/log.py index 3080daee2..f8c0ba68d 100644 --- a/nova/log.py +++ b/nova/log.py @@ -319,7 +319,7 @@ def audit(msg, *args, **kwargs): class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" - def __init__(self, logger, level=logging.DEBUG): + def __init__(self, logger, level=logging.INFO): self.logger = logger self.level = level diff --git a/nova/service.py b/nova/service.py index ec2de9004..4c7d1adc4 100644 --- a/nova/service.py +++ b/nova/service.py @@ -249,7 +249,7 @@ class WSGIService(object): def _find_config(self): """Attempt to find 'paste' configuration file.""" location = wsgi.paste_config_file(self._config_name) - logging.debug(_("Using paste.deploy config at: %s"), location) + logging.info(_("Using paste.deploy config: %s"), location) return location def _load_config(self): diff --git a/nova/wsgi.py b/nova/wsgi.py index 4da8eff82..82ba006c8 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -77,10 +77,7 @@ class Server(object): def wait(self): """Wait until server has been stopped.""" - try: - self._server.wait() - except KeyboardInterrupt: - pass + self._server.wait() class Request(webob.Request): From 21d3bb4e034e3cdb57247e373cd62f123be74c7d Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 18:00:38 -0400 Subject: [PATCH 031/107] Cleanup of the cleanup. --- bin/nova-api | 6 ++---- setup.cfg | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index ff4aa83d1..1fcda24f4 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,4 +1,5 @@ #!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -18,13 +19,10 @@ """Starter script for Nova API. -Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings -about re-imports should be ignored. +Starts both the EC2 and OpenStack APIs in separate processes. """ -# pylint: disable=W0404 - import sys import multiprocessing diff --git a/setup.cfg b/setup.cfg index 7efc46f23..9c0a331e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[develop] -user = True - [build_sphinx] all_files = 1 build-dir = doc/build From 1a2932dd425db0792c380a987805401177a55bf8 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 20:18:29 -0400 Subject: [PATCH 032/107] Introduced Loader concept, for paste decouple. --- bin/nova-api | 2 + nova/exception.py | 8 +++ nova/service.py | 38 +++++------- nova/wsgi.py | 149 ++++++++++++++++++++++++++-------------------- 4 files changed, 110 insertions(+), 87 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 1fcda24f4..885fb0ba4 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,6 +51,8 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() + +# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) diff --git a/nova/exception.py b/nova/exception.py index f3a452228..0fcd3de95 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -589,3 +589,11 @@ class MigrationError(NovaException): class MalformedRequestBody(NovaException): message = _("Malformed message body: %(reason)s") + + +class PasteConfigNotFound(NotFound): + message = _("Could not find paste config at %(path)s") + + +class PasteAppNotFound(NotFound): + message = _("Could not load paste app '%(name)s' from %(path)s") diff --git a/nova/service.py b/nova/service.py index 4c7d1adc4..e27d61371 100644 --- a/nova/service.py +++ b/nova/service.py @@ -235,34 +235,24 @@ class Service(object): class WSGIService(object): """Provides ability to launch API from a 'paste' configuration.""" - def __init__(self, name, config_name=None): - """Initialize, but do not start, an API service.""" + def __init__(self, name, loader=None): + """Initialize, but do not start the WSGI service. + + :param name: The name of the WSGI service given to the loader. + :param loader: Loads the WSGI application using the given name. + :returns: None + + """ self.name = name - self._config_name = config_name or FLAGS.api_paste_config - self._config_location = self._find_config() - self._config = self._load_config() - self.application = self._load_application() - host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") - port = getattr(FLAGS, '%s_listen_port' % name, 0) - self.server = wsgi.Server(name, self.application, host, port) - - def _find_config(self): - """Attempt to find 'paste' configuration file.""" - location = wsgi.paste_config_file(self._config_name) - logging.info(_("Using paste.deploy config: %s"), location) - return location - - def _load_config(self): - """Read and return the 'paste' configuration file.""" - return wsgi.load_paste_configuration(self._config_location, self.name) - - def _load_application(self): - """Using the loaded configuration, return the WSGI application.""" - return wsgi.load_paste_app(self._config_location, self.name) + self.loader = loader or wsgi.Loader() + self.application = self.loader.load_app(name) + self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + self.port = getattr(FLAGS, '%s_listen_port' % name, 0) + self.server = wsgi.Server(name, self.application) def start(self): """Start serving this API using loaded configuration.""" - self.server.start() + self.server.start(self.host, self.port) def stop(self): """Stop serving this API.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index 82ba006c8..328dc083f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -35,48 +35,78 @@ from paste import deploy from nova import exception from nova import flags -from nova import log as logging +from nova import log from nova import utils FLAGS = flags.FLAGS -LOG = logging.getLogger('nova.wsgi') +LOG = log.getLogger('nova.wsgi') class Server(object): """Server class to manage multiple WSGI sockets and applications.""" default_pool_size = 1000 - logger_name = "eventlet.wsgi.server" - def __init__(self, name, app, host, port, pool_size=None): + def __init__(self, name, app, pool_size=None): + """Initialize, but do not start, a WSGI server. + + :param name: The name to use for logging and other human purposes. + :param app: The WSGI application to serve. + :param pool_size: Maximum number of eventlets to spawn concurrently. + :returns: None + + """ self.name = name self.app = app - self.host = host - self.port = port - self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) - self._log = logging.WritableLogger(logging.getLogger(self.logger_name)) + self.pool_size = pool_size or self.default_pool_size def _start(self, socket): - """Blocking eventlet WSGI server launched from the real 'start'.""" - eventlet.wsgi.server(socket, - self.app, - custom_pool=self._pool, - log=self._log) + """Run the blocking eventlet WSGI server. - def start(self, backlog=128): - """Serve given WSGI application using the given parameters.""" - socket = eventlet.listen((self.host, self.port), backlog=backlog) + :param socket: The socket where the WSGI server will serve it's app. + :returns: None + + """ + pool = eventlet.GreenPool(self.pool_size) + logger = log.WritableLogger(log.getLogger("eventlet.wsgi.server")) + eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) + + def start(self, host, port, backlog=128): + """Start serving a WSGI application. + + :param host: IP address to serve the application. + :param port: Port number to server the application. + :param backlog: Maximum number of queued connections. + :returns: None + + """ + socket = eventlet.listen((host, port), backlog=backlog) self._server = eventlet.spawn(self._start, socket) (self.host, self.port) = socket.getsockname() - LOG.info(_('Starting %(name)s on %(host)s:%(port)s') % self.__dict__) + LOG.info(_("Started %(name)s on %(host)s:%(port)s") % self.__dict__) def stop(self): - """Stop this server by killing the greenthread running it.""" + """Stop this server. + + This is not a very nice action, as currently the method by which a + server is stopped is by killing it's eventlet. + + :returns: None + + """ + LOG.debug(_("Stopping WSGI server.")) self._server.kill() def wait(self): - """Wait until server has been stopped.""" + """Block, until the server has stopped. + + Waits on the server's eventlet to finish, then returns. + + :returns: None + + """ + LOG.debug(_("Waiting for WSGI server to stop.")) self._server.wait() @@ -305,57 +335,50 @@ class Router(object): return app -def paste_config_file(basename): - """Find the best location in the system for a paste config file. +class Loader(object): + """Used to load WSGI applications from paste configurations.""" - Search Order - ------------ + def __init__(self, config_path=None): + """Initialize the loader, and attempt to find the config. - The search for a paste config file honors `FLAGS.state_path`, which in a - version checked out from bzr will be the `nova` directory in the top level - of the checkout, and in an installation for a package for your distribution - will likely point to someplace like /etc/nova. + :param config_path: Full or relative path to the paste config. + :returns: None - This method tries to load places likely to be used in development or - experimentation before falling back to the system-wide configuration - in `/etc/nova/`. + """ + config_path = config_path or FLAGS.api_paste_config + self.config_path = self._find_config(config_path) - * Current working directory - * the `etc` directory under state_path, because when working on a checkout - from bzr this will point to the default - * top level of FLAGS.state_path, for distributions - * /etc/nova, which may not be diffrerent from state_path on your distro + def _find_config(self, config_path): + """Find the paste configuration file using the given hint. - """ - configfiles = [basename, - os.path.join(FLAGS.state_path, 'etc', 'nova', basename), - os.path.join(FLAGS.state_path, 'etc', basename), - os.path.join(FLAGS.state_path, basename), - '/etc/nova/%s' % basename] - for configfile in configfiles: - if os.path.exists(configfile): - return configfile + :param config_path: Full or relative path to the paste config. + :returns: Full path of the paste config, if it exists. + :raises: `nova.exception.PasteConfigNotFound` - raise Exception(_("Unable to find paste.deploy config '%s'") % basename) + """ + possible_locations = [ + config_path, + os.path.join(FLAGS.state_path, "etc", "nova", config_path), + os.path.join(FLAGS.state_path, "etc", config_path), + os.path.join(FLAGS.state_path, config_path), + "/etc/nova/%s" % config_path, + ] + for path in possible_locations: + if os.path.exists(path): + return os.path.abspath(path) -def load_paste_configuration(filename, appname): - """Returns a paste configuration dict, or None.""" - filename = os.path.abspath(filename) - config = None - try: - config = deploy.appconfig('config:%s' % filename, name=appname) - except LookupError: - pass - return config + raise exception.PasteConfigNotFound(path=os.path.abspath(config_path)) + def load_app(self, name): + """Return the paste URLMap wrapped WSGI application. -def load_paste_app(filename, appname): - """Builds a wsgi app from a paste config, None if app not configured.""" - filename = os.path.abspath(filename) - app = None - try: - app = deploy.loadapp('config:%s' % filename, name=appname) - except LookupError: - pass - return app + :param name: Name of the application to load. + :returns: Paste URLMap object wrapping the requested application. + :raises: `nova.exception.PasteAppNotFound` + + """ + app = deploy.loadapp("config:%s" % self.config_path, name=name) + if app is None: + raise exception.PasteAppNotFound(name=name, path=self.config_path) + return app From 6c8f65f97b81f36cd42feab81d103432b48f779c Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 20:26:47 -0400 Subject: [PATCH 033/107] Cleanup. --- nova/service.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/nova/service.py b/nova/service.py index e27d61371..bb38cddb6 100644 --- a/nova/service.py +++ b/nova/service.py @@ -251,15 +251,31 @@ class WSGIService(object): self.server = wsgi.Server(name, self.application) def start(self): - """Start serving this API using loaded configuration.""" + """Start serving this service using loaded configuration. + + Also, retrieve updated port number in case '0' was passed in, which + indicates a random port should be used. + + :returns: None + + """ self.server.start(self.host, self.port) + self.port = self.server.port def stop(self): - """Stop serving this API.""" + """Stop serving this API. + + :returns: None + + """ self.server.stop() def wait(self): - """Wait for the service to stop serving this API.""" + """Wait for the service to stop serving this API. + + :returns: None + + """ self.server.wait() From badd475f92b717999645811153ec89a89c7b2443 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 21:28:51 -0400 Subject: [PATCH 034/107] Added tests for WSGI loader. --- nova/wsgi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 328dc083f..ce0caf65b 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -378,7 +378,8 @@ class Loader(object): :raises: `nova.exception.PasteAppNotFound` """ - app = deploy.loadapp("config:%s" % self.config_path, name=name) - if app is None: + try: + return deploy.loadapp("config:%s" % self.config_path, name=name) + except LookupError as err: + LOG.error(err) raise exception.PasteAppNotFound(name=name, path=self.config_path) - return app From 9ef2b8f157877c0b7ad42c31fd482df08c7f5322 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 10:12:43 -0400 Subject: [PATCH 035/107] Cleanup and addition of tests for WSGI server. --- bin/nova-api | 2 +- nova/utils.py | 2 -- nova/wsgi.py | 18 ++++++++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 885fb0ba4..89112a159 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# + # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/nova/utils.py b/nova/utils.py index 22d0ce940..691134ada 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -238,8 +238,6 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' - if not os.path.exists(filename): - return flagfile = ['--flagfile=%s' % filename] sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] diff --git a/nova/wsgi.py b/nova/wsgi.py index ce0caf65b..8a7d38252 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -21,16 +21,16 @@ import os import sys + from xml.dom import minidom import eventlet import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True, time=True) -import routes +import greenlet import routes.middleware -import webob import webob.dec import webob.exc + from paste import deploy from nova import exception @@ -39,6 +39,9 @@ from nova import log from nova import utils +eventlet.patcher.monkey_patch(all=False, socket=True, time=True) + + FLAGS = flags.FLAGS LOG = log.getLogger('nova.wsgi') @@ -95,7 +98,7 @@ class Server(object): :returns: None """ - LOG.debug(_("Stopping WSGI server.")) + LOG.info(_("Stopping WSGI server.")) self._server.kill() def wait(self): @@ -106,8 +109,11 @@ class Server(object): :returns: None """ - LOG.debug(_("Waiting for WSGI server to stop.")) - self._server.wait() + LOG.info(_("Waiting for WSGI server to stop.")) + try: + self._server.wait() + except greenlet.GreenletExit: + LOG.info(_("WSGI server has stopped.")) class Request(webob.Request): From e78007b70f2adf6dbd392265b7ca2f42502ad31d Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 14:54:53 -0400 Subject: [PATCH 036/107] pep8 fixes --- bin/nova-api | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 89112a159..2345b3f2c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,8 +51,6 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() - -# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) From 80ba340e9e8a7f3742813322c936d6b15b6b410f Mon Sep 17 00:00:00 2001 From: John Tran Date: Mon, 20 Jun 2011 15:28:34 -0700 Subject: [PATCH 037/107] nova-manage checks if user is member of proj, prior to adding role for that project --- bin/nova-manage | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index dbdb798a7..7226fcfa1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -257,6 +257,11 @@ class RoleCommands(object): """adds role to user if project is specified, adds project specific role arguments: user, role [project]""" + if project: + projobj = self.manager.get_project(project) + if not projobj.has_member(user): + print "%s not a member of %s" % (user, project) + return self.manager.add_role(user, role, project) def has(self, user, role, project=None): From 0e033cfab0a3e987590abca970ecd895b32761cf Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 19:32:18 -0400 Subject: [PATCH 038/107] Removed logging logic from __init__, added concept of Launcher...no tests for it yet. --- bin/nova-api | 38 +++++++------------------- nova/__init__.py | 10 +------ nova/service.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++- nova/wsgi.py | 1 - 4 files changed, 80 insertions(+), 39 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 2345b3f2c..563d7c090 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,44 +24,26 @@ Starts both the EC2 and OpenStack APIs in separate processes. """ import sys -import multiprocessing -import nova.flags import nova.log import nova.service import nova.version -import nova.utils - - -def launch(service_name): - """Launch WSGI service with name matching 'paste' config file section.""" - service = nova.service.WSGIService(service_name) - service.start() - try: - service.wait() - except KeyboardInterrupt: - service.stop() def main(): - """Begin process of launching both EC2 and OSAPI services.""" - version = nova.version.version_string_with_vcs() - logger = nova.log.getLogger("nova.api") - logger.audit(_("Starting nova-api node (version %s)") % version) - - nova.flags.FLAGS(sys.argv) - nova.utils.default_flagfile() - - pool = multiprocessing.Pool(2) - pool.map_async(launch, ["ec2", "osapi"]) - pool.close() + """Launch EC2 and OSAPI services.""" + ec2 = nova.service.WSGIService("ec2") + osapi = nova.service.WSGIService("osapi") + launcher = nova.service.Launcher(sys.argv) + launcher.launch_service(ec2) + launcher.launch_service(osapi) + try: - pool.join() + launcher.wait() except KeyboardInterrupt: - logger.audit(_("Exiting...")) - pool.terminate() - + launcher.stop() + if __name__ == '__main__': sys.exit(main()) diff --git a/nova/__init__.py b/nova/__init__.py index 886eaee20..884c4a713 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -33,13 +33,5 @@ import gettext -import log as logging - -def initialize(): - gettext.install("nova", unicode=1) - logging.setup() - logging.debug(_("Initialized logging.")) - - -initialize() +gettext.install("nova", unicode=1) diff --git a/nova/service.py b/nova/service.py index bb38cddb6..d896de1fe 100644 --- a/nova/service.py +++ b/nova/service.py @@ -19,10 +19,12 @@ """Generic Node baseclass for all workers that run on hosts.""" -import greenlet import inspect +import multiprocessing import os +import greenlet + from eventlet import greenthread from nova import context @@ -53,6 +55,72 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", 'File name for the paste.deploy config for nova-api') +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self, _flags=None): + """Initialize the service launcher. + + :param _flags: Flags to use for the services we're going to load. + :returns: None + + """ + self._services = [] + self._version = version.version_string_with_vcs() + logging.setup() + logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) + FLAGS(_flags) + utils.default_flagfile() + + + @staticmethod + def run_service(service): + """Start and wait for a service to finish. + + :param service: Service to run and wait for. + :returns: None + + """ + service.start() + try: + service.wait() + except KeyboardInterrupt: + service.stop() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + process = multiprocessing.Process(target=self.run_service, + args=(service,)) + process.start() + self._services.append(process) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + for service in self._services: + if service.is_alive(): + service.terminate() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + for service in self._services: + service.join() + logging.info("Process exited with %d" % service.exitcode) + + + class Service(object): """Base class for workers that run on hosts.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index 8a7d38252..097fa4734 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -109,7 +109,6 @@ class Server(object): :returns: None """ - LOG.info(_("Waiting for WSGI server to stop.")) try: self._server.wait() except greenlet.GreenletExit: From 9a25bc281096f684f358989209117e4750a235fa Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:21:33 -0400 Subject: [PATCH 039/107] Tests for WSGI/Launcher --- bin/nova-api | 4 ++-- nova/service.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 563d7c090..20ad4bfa5 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,12 +38,12 @@ def main(): launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) - + try: launcher.wait() except KeyboardInterrupt: launcher.stop() - + if __name__ == '__main__': sys.exit(main()) diff --git a/nova/service.py b/nova/service.py index d896de1fe..08071e80c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -69,9 +69,8 @@ class Launcher(object): self._version = version.version_string_with_vcs() logging.setup() logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) - FLAGS(_flags) utils.default_flagfile() - + FLAGS(_flags or []) @staticmethod def run_service(service): @@ -120,7 +119,6 @@ class Launcher(object): logging.info("Process exited with %d" % service.exitcode) - class Service(object): """Base class for workers that run on hosts.""" From ecde6570b1a0b3142746a309a3e6177892914c54 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:32:31 -0400 Subject: [PATCH 040/107] Removed unneeded import. --- bin/nova-api | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 20ad4bfa5..b94928c7b 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -27,7 +27,6 @@ import sys import nova.log import nova.service -import nova.version def main(): From a2e83c39d0a9c590d7950aea6feee49b41abb24e Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:35:45 -0400 Subject: [PATCH 041/107] Removed debugging and switched eventlet to monkey patch everything. --- nova/service.py | 1 - nova/wsgi.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/service.py b/nova/service.py index 08071e80c..4cf372377 100644 --- a/nova/service.py +++ b/nova/service.py @@ -116,7 +116,6 @@ class Launcher(object): """ for service in self._services: service.join() - logging.info("Process exited with %d" % service.exitcode) class Service(object): diff --git a/nova/wsgi.py b/nova/wsgi.py index 097fa4734..3c4f8a5da 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -39,7 +39,7 @@ from nova import log from nova import utils -eventlet.patcher.monkey_patch(all=False, socket=True, time=True) +eventlet.patcher.monkey_patch() FLAGS = flags.FLAGS From 1753a6256f0cc56befbc2f0342795a619625b27f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:54:14 -0400 Subject: [PATCH 042/107] log -> logging to keep with convention --- nova/wsgi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 3c4f8a5da..2991d1529 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -35,7 +35,7 @@ from paste import deploy from nova import exception from nova import flags -from nova import log +from nova import log as logging from nova import utils @@ -43,7 +43,7 @@ eventlet.patcher.monkey_patch() FLAGS = flags.FLAGS -LOG = log.getLogger('nova.wsgi') +LOG = logging.getLogger('nova.wsgi') class Server(object): @@ -72,7 +72,7 @@ class Server(object): """ pool = eventlet.GreenPool(self.pool_size) - logger = log.WritableLogger(log.getLogger("eventlet.wsgi.server")) + logger = logging.WritableLogger(logging.getLogger("eventlet.wsgi")) eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) def start(self, host, port, backlog=128): From b2345353404003f2718a43f145a0d7e80b202486 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 09:46:00 -0400 Subject: [PATCH 043/107] Monkey patching 'os' kills multiprocessing's .join() functionality. Also, messed up the name of the eventlet WSGI logger. --- nova/wsgi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 2991d1529..0c869cfb8 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -39,7 +39,7 @@ from nova import log as logging from nova import utils -eventlet.patcher.monkey_patch() +eventlet.patcher.monkey_patch(socket=True, time=True) FLAGS = flags.FLAGS @@ -72,7 +72,8 @@ class Server(object): """ pool = eventlet.GreenPool(self.pool_size) - logger = logging.WritableLogger(logging.getLogger("eventlet.wsgi")) + log_name = "eventlet.wsgi.server" + logger = logging.WritableLogger(logging.getLogger(log_name)) eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) def start(self, host, port, backlog=128): From d2cd709ffbfa4822c1dd1e2c1f80194425b794d5 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 10:35:50 -0400 Subject: [PATCH 044/107] Oops, I broke --help on nova-api, fixed now. --- nova/service.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nova/service.py b/nova/service.py index 4cf372377..f82c64d9c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -67,10 +67,31 @@ class Launcher(object): """ self._services = [] self._version = version.version_string_with_vcs() + self._flags = _flags + self._setup_logging() + self._setup_flags() + + def _setup_logging(self): + """Logic to ensure logging is going to work correctly for services. + + :returns: None + + """ logging.setup() logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) + + def _setup_flags(self): + """Logic to ensure flags/configuration are correctly set. + + :returns: None + + """ utils.default_flagfile() - FLAGS(_flags or []) + FLAGS(self._flags or []) + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() @staticmethod def run_service(service): From 43bf0333e972089be1367566b4e53969f7c979dc Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 11:25:44 -0400 Subject: [PATCH 045/107] Very small alterations, switched from using start() to pass host/port, to just defining them up front in init. Doesn't make sense to set them in start because we can't start more than once any way. Also, unbroke binaries. --- bin/nova-ajax-console-proxy | 5 +++-- bin/nova-direct-api | 7 +++++-- bin/nova-objectstore | 7 +++++-- bin/nova-vncproxy | 7 +++++-- nova/service.py | 13 ++++++++----- nova/wsgi.py | 37 +++++++++++++++++++++---------------- 6 files changed, 47 insertions(+), 29 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index d88f59e40..21cf68007 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -137,8 +137,9 @@ if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) logging.setup() - server = wsgi.Server() + acp_port = FLAGS.ajax_console_proxy_port acp = AjaxConsoleProxy() acp.register_listeners() - server.start(acp, FLAGS.ajax_console_proxy_port, host='0.0.0.0') + server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port) + server.start() server.wait() diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 83ec72722..5d63eb87f 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,6 +93,9 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server() - server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host) + server = wsgi.Server("Direct API", + with_auth, + host=FLAGS.direct_host, + port=FLAGS.direct_port) + server.start() server.wait() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 6ef841b85..aa0dc063f 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,6 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server() - server.start(router, FLAGS.s3_port, host=FLAGS.s3_host) + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, + host=FLAGS.s3_host) + server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index ccb97e3a3..62d3b948c 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,6 +96,9 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server() - server.start(with_auth, FLAGS.vncproxy_port, host=FLAGS.vncproxy_host) + server = wsgi.Server("VNC Proxy", + with_auth, + host=FLAGS.vncproxy_host, + port=FLAGS.vncproxy_port) + server.start() server.wait() diff --git a/nova/service.py b/nova/service.py index f82c64d9c..854b4572d 100644 --- a/nova/service.py +++ b/nova/service.py @@ -331,10 +331,13 @@ class WSGIService(object): """ self.name = name self.loader = loader or wsgi.Loader() - self.application = self.loader.load_app(name) - self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") - self.port = getattr(FLAGS, '%s_listen_port' % name, 0) - self.server = wsgi.Server(name, self.application) + self.app = self.loader.load_app(name) + self.host = getattr(FLAGS, '%s_listen' % name, None) + self.port = getattr(FLAGS, '%s_listen_port' % name, None) + self.server = wsgi.Server(name, + self.app, + host=self.host, + port=self.port) def start(self): """Start serving this service using loaded configuration. @@ -345,7 +348,7 @@ class WSGIService(object): :returns: None """ - self.server.start(self.host, self.port) + self.server.start() self.port = self.server.port def stop(self): diff --git a/nova/wsgi.py b/nova/wsgi.py index 0c869cfb8..23d29079f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -47,47 +47,52 @@ LOG = logging.getLogger('nova.wsgi') class Server(object): - """Server class to manage multiple WSGI sockets and applications.""" + """Server class to manage a WSGI server, serving a WSGI application.""" default_pool_size = 1000 - def __init__(self, name, app, pool_size=None): + def __init__(self, name, app, host=None, port=None, pool_size=None): """Initialize, but do not start, a WSGI server. - :param name: The name to use for logging and other human purposes. + :param name: Pretty name for logging. :param app: The WSGI application to serve. + :param host: IP address to serve the application. + :param port: Port number to server the application. :param pool_size: Maximum number of eventlets to spawn concurrently. :returns: None """ self.name = name self.app = app - self.pool_size = pool_size or self.default_pool_size + self.host = host or "0.0.0.0" + self.port = port or 0 + self._server = None + self._socket = None + self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) + self._logger = logging.getLogger("eventlet.wsgi.server") + self._wsgi_logger = logging.WritableLogger(self._logger) - def _start(self, socket): + def _start(self): """Run the blocking eventlet WSGI server. - :param socket: The socket where the WSGI server will serve it's app. :returns: None """ - pool = eventlet.GreenPool(self.pool_size) - log_name = "eventlet.wsgi.server" - logger = logging.WritableLogger(logging.getLogger(log_name)) - eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) + eventlet.wsgi.server(self._socket, + self.app, + custom_pool=self._pool, + log=self._wsgi_logger) - def start(self, host, port, backlog=128): + def start(self, backlog=128): """Start serving a WSGI application. - :param host: IP address to serve the application. - :param port: Port number to server the application. :param backlog: Maximum number of queued connections. :returns: None """ - socket = eventlet.listen((host, port), backlog=backlog) - self._server = eventlet.spawn(self._start, socket) - (self.host, self.port) = socket.getsockname() + self._socket = eventlet.listen((self.host, self.port), backlog=backlog) + self._server = eventlet.spawn(self._start) + (self.host, self.port) = self._socket.getsockname() LOG.info(_("Started %(name)s on %(host)s:%(port)s") % self.__dict__) def stop(self): From 57268b17c8cb3483e7daafb20f8e646b6b23ac3a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 11:50:28 -0400 Subject: [PATCH 046/107] Fixed objectstore test. --- nova/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/service.py b/nova/service.py index 854b4572d..ca247c0f9 100644 --- a/nova/service.py +++ b/nova/service.py @@ -332,8 +332,8 @@ class WSGIService(object): self.name = name self.loader = loader or wsgi.Loader() self.app = self.loader.load_app(name) - self.host = getattr(FLAGS, '%s_listen' % name, None) - self.port = getattr(FLAGS, '%s_listen_port' % name, None) + self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + self.port = getattr(FLAGS, '%s_listen_port' % name, 0) self.server = wsgi.Server(name, self.app, host=self.host, From 1345ed5da6400e5f84ba58de4e30e7f5301b40f7 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 12:03:27 -0400 Subject: [PATCH 047/107] Removed whitespace. --- bin/nova-direct-api | 4 ++-- bin/nova-objectstore | 6 +++--- bin/nova-vncproxy | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 5d63eb87f..c6cf9b2ff 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,8 +93,8 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server("Direct API", - with_auth, + server = wsgi.Server("Direct API", + with_auth, host=FLAGS.direct_host, port=FLAGS.direct_port) server.start() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index aa0dc063f..1aef3a255 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,9 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server("S3 Objectstore", - router, - port=FLAGS.s3_port, + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, host=FLAGS.s3_host) server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index 62d3b948c..72271df3a 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,7 +96,7 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server("VNC Proxy", + server = wsgi.Server("VNC Proxy", with_auth, host=FLAGS.vncproxy_host, port=FLAGS.vncproxy_port) From 9fba8dde1910dac38b51a14dabfba875d373587c Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 22 Jun 2011 09:33:46 -0400 Subject: [PATCH 048/107] fix some issues with flags and logging --- nova/service.py | 19 ++++++++++++++----- nova/utils.py | 10 ++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/nova/service.py b/nova/service.py index ca247c0f9..41c6551e0 100644 --- a/nova/service.py +++ b/nova/service.py @@ -38,6 +38,8 @@ from nova import version from nova import wsgi +LOG = logging.getLogger('nova.service') + FLAGS = flags.FLAGS flags.DEFINE_integer('report_interval', 10, 'seconds between nodes reporting state to datastore', @@ -58,18 +60,19 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", class Launcher(object): """Launch one or more services and wait for them to complete.""" - def __init__(self, _flags=None): + def __init__(self, flags=None): """Initialize the service launcher. - :param _flags: Flags to use for the services we're going to load. + :param flags: Flags to use for the services we're going to load. :returns: None """ self._services = [] self._version = version.version_string_with_vcs() - self._flags = _flags - self._setup_logging() + self._flags = flags self._setup_flags() + self._setup_logging() + self._log_flags() def _setup_logging(self): """Logic to ensure logging is going to work correctly for services. @@ -86,13 +89,19 @@ class Launcher(object): :returns: None """ - utils.default_flagfile() + utils.default_flagfile(args=self._flags) FLAGS(self._flags or []) flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) FLAGS.ParseNewFlags() + def _log_flags(self): + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) + @staticmethod def run_service(service): """Start and wait for a service to finish. diff --git a/nova/utils.py b/nova/utils.py index e2ac16f31..a9b0f3128 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -226,8 +226,10 @@ def novadir(): return os.path.abspath(nova.__file__).split('nova/__init__.pyc')[0] -def default_flagfile(filename='nova.conf'): - for arg in sys.argv: +def default_flagfile(filename='nova.conf', args=None): + if args is None: + args = sys.argv + for arg in args: if arg.find('flagfile') != -1: break else: @@ -239,8 +241,8 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' - flagfile = ['--flagfile=%s' % filename] - sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] + flagfile = '--flagfile=%s' % filename + args.insert(1, flagfile) def debug(arg): From 4103c3796f06cb2d6414ada4e68bc52fb1793705 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 22 Jun 2011 16:14:01 -0500 Subject: [PATCH 049/107] Adding backup rotation --- nova/exception.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/exception.py b/nova/exception.py index f3a452228..a548a638c 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -549,6 +549,10 @@ class GlobalRoleNotAllowed(NotAllowed): message = _("Unable to use global role %(role_id)s") +class ImageRotationNotAllowed(NovaException): + message = _("Rotation is not allowed for snapshots") + + #TODO(bcwaldon): EOL this exception! class Duplicate(NovaException): pass From 86c6a27eb001360206f8bca0aceaeb27dbd74273 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 22 Jun 2011 23:27:49 -0400 Subject: [PATCH 050/107] run launcher first since it initializes global flags and logging --- bin/nova-api | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index b94928c7b..121f6f9a0 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -31,10 +31,11 @@ import nova.service def main(): """Launch EC2 and OSAPI services.""" + launcher = nova.service.Launcher(sys.argv) + ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") - launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) From e90585650eec92defa9b56fadbe8e5c911173d17 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 01:22:50 -0700 Subject: [PATCH 051/107] Add some resource checking for memory available when scheduling Various changes to d-sched to plan for scheduling on different topics, which cleans up some of the resource checking. Re-compute weights when building more than 1 instance, accounting for resources that would be consumed. --- nova/scheduler/host_filter.py | 10 ++- nova/scheduler/least_cost.py | 39 +++++++---- nova/scheduler/zone_aware_scheduler.py | 92 +++++++++++++++++++++----- 3 files changed, 106 insertions(+), 35 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index bd6b26608..818ae4a30 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -305,8 +305,11 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler): 'instance_type': } """ - def filter_hosts(self, num, request_spec): + def filter_hosts(self, topic, request_spec, hosts): """Filter the full host list (from the ZoneManager)""" + + if hosts: + return hosts filter_name = request_spec.get('filter', None) host_filter = choose_host_filter(filter_name) @@ -317,8 +320,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler): name, query = host_filter.instance_type_to_filter(instance_type) return host_filter.filter_hosts(self.zone_manager, query) - def weigh_hosts(self, num, request_spec, hosts): + def weigh_hosts(self, topic, request_spec, hosts): """Derived classes must override this method and return a lists of hosts in [{weight, hostname}] format. """ - return [dict(weight=1, hostname=host) for host, caps in hosts] + return [dict(weight=1, hostname=hostname, capabilities=caps) + for hostname, caps in hosts] diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py index 629fe2e42..72db2fd1b 100644 --- a/nova/scheduler/least_cost.py +++ b/nova/scheduler/least_cost.py @@ -48,25 +48,36 @@ def noop_cost_fn(host): return 1 -flags.DEFINE_integer('fill_first_cost_fn_weight', 1, +flags.DEFINE_integer('compute_fill_first_cost_fn_weight', 1, 'How much weight to give the fill-first cost function') -def fill_first_cost_fn(host): +def compute_fill_first_cost_fn(host): """Prefer hosts that have less ram available, filter_hosts will exclude hosts that don't have enough ram""" hostname, caps = host - free_mem = caps['compute']['host_memory_free'] + free_mem = caps['host_memory_free'] return free_mem class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler): - def get_cost_fns(self): + def __init__(self, *args, **kwargs): + self.cost_fns_cache = {} + super(LeastCoastScheduler, self).__init__(*args, **kwargs) + + def get_cost_fns(self, topic): """Returns a list of tuples containing weights and cost functions to use for weighing hosts """ + + if topic in self.cost_fns_cache: + return self.cost_fns_cache[topic] + cost_fns = [] for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions: + if not cost_fn_str.startswith('%s_' % topic) and \ + not cost_fn_str.startswith('noop'): + continue try: # NOTE(sirp): import_class is somewhat misnamed since it can @@ -84,23 +95,23 @@ class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler): cost_fns.append((weight, cost_fn)) + self.cost_fns_cache[topic] = cost_fns return cost_fns - def weigh_hosts(self, num, request_spec, hosts): + def weigh_hosts(self, topic, request_spec, hosts): """Returns a list of dictionaries of form: - [ {weight: weight, hostname: hostname} ]""" + [ {weight: weight, hostname: hostname, capabilities: capabs} ] + """ - # FIXME(sirp): weigh_hosts should handle more than just instances - hostnames = [hostname for hostname, caps in hosts] - - cost_fns = self.get_cost_fns() + cost_fns = self.get_cost_fns(topic) costs = weighted_sum(domain=hosts, weighted_fns=cost_fns) weighted = [] weight_log = [] - for cost, hostname in zip(costs, hostnames): + for cost, (hostname, caps) in zip(costs, hosts): weight_log.append("%s: %s" % (hostname, "%.2f" % cost)) - weight_dict = dict(weight=cost, hostname=hostname) + weight_dict = dict(weight=cost, hostname=hostname, + capabilities=caps) weighted.append(weight_dict) LOG.debug(_("Weighted Costs => %s") % weight_log) @@ -127,7 +138,8 @@ def weighted_sum(domain, weighted_fns, normalize=True): weighted_fns - list of weights and functions like: [(weight, objective-functions)] - Returns an unsorted of scores. To pair with hosts do: zip(scores, hosts) + Returns an unsorted list of scores. To pair with hosts do: + zip(scores, hosts) """ # Table of form: # { domain1: [score1, score2, ..., scoreM] @@ -150,7 +162,6 @@ def weighted_sum(domain, weighted_fns, normalize=True): domain_scores = [] for idx in sorted(score_table): elem_score = sum(score_table[idx]) - elem = domain[idx] domain_scores.append(elem_score) return domain_scores diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index e7bff2faa..d4d3d0414 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -224,18 +224,34 @@ class ZoneAwareScheduler(driver.Scheduler): raise NotImplemented(_("Zone Aware Scheduler only understands " "Compute nodes (for now)")) - #TODO(sandy): how to infer this from OS API params? - num_instances = 1 + num_instances = request_spec['num_instances'] + instance_type = request_spec['instance_type'] - # Filter local hosts based on requirements ... - host_list = self.filter_hosts(num_instances, request_spec) + weighted = [] + host_list = None - # TODO(sirp): weigh_hosts should also be a function of 'topic' or - # resources, so that we can apply different objective functions to it + for i in xrange(num_instances): + # Filter local hosts based on requirements ... + # + # The first pass through here will pass 'None' as the + # host_list.. which tells the filter to build the full + # list of hosts. + # On a 2nd pass, the filter can modify the host_list with + # any updates it needs to make based on resources that + # may have been consumed from a previous build.. + host_list = self.filter_hosts(topic, request_spec, host_list) + if not host_list: + break - # then weigh the selected hosts. - # weighted = [{weight=weight, name=hostname}, ...] - weighted = self.weigh_hosts(num_instances, request_spec, host_list) + # then weigh the selected hosts. + # weighted = [{weight=weight, hostname=hostname, + # capabilities=capabs}, ...] + weights = self.weigh_hosts(topic, request_spec, host_list) + weights.sort(key=operator.itemgetter('weight')) + best_weight = weights[0] + weighted.append(best_weight) + self.consume_resources(best_weight['capabilities'], + instance_type) # Next, tack on the best weights from the child zones ... json_spec = json.dumps(request_spec) @@ -254,18 +270,58 @@ class ZoneAwareScheduler(driver.Scheduler): weighted.sort(key=operator.itemgetter('weight')) return weighted - def filter_hosts(self, num, request_spec): - """Derived classes must override this method and return - a list of hosts in [(hostname, capability_dict)] format. + def compute_filter(self, hostname, capabilities, request_spec): + """Return whether or not we can schedule to this compute node. + Derived classes should override this and return True if the host + is acceptable for scheduling. """ - # NOTE(sirp): The default logic is the equivalent to AllHostsFilter - service_states = self.zone_manager.service_states - return [(host, services) - for host, services in service_states.iteritems()] + instance_type = request_spec['instance_type'] + reqested_mem = instance_type['memory_mb'] + return capabilities['host_memory_free'] >= requested_mem - def weigh_hosts(self, num, request_spec, hosts): + def filter_hosts(self, topic, request_spec, host_list=None): + """Return a list of hosts which are acceptable for scheduling. + Return value should be a list of (hostname, capability_dict)s. + Derived classes may override this, but may find the + '_filter' function more appropriate. + """ + + def _default_filter(self, hostname, capabilities, request_spec): + """Default filter function if there's no _filter""" + # NOTE(sirp): The default logic is the equivalent to + # AllHostsFilter + return True + + filter_func = getattr(self, '%s_filter' % topic, _default_filter) + + filtered_hosts = [] + if host_list is None: + host_list = self.zone_manager.service_states.iteritems() + for host, services in host_list: + if topic not in services: + continue + if filter_func(host, services['topic'], request_spec): + filtered_hosts.append((host, services['topic'])) + + def weigh_hosts(self, topic, request_spec, hosts): """Derived classes may override this to provide more sophisticated scheduling objectives """ # NOTE(sirp): The default logic is the same as the NoopCostFunction - return [dict(weight=1, hostname=host) for host, caps in hosts] + return [dict(weight=1, hostname=hostname, capabilities=capabilities) + for hostname, capabilities in hosts] + + def compute_consume(self, capabilities, instance_type): + """Consume compute resources for selected host""" + + requested_mem = max(instance_type['memory_mb'], 0) + capabilities['host_memory_free'] -= requested_mem + + def consume_resources(self, topic, capabilities, instance_type): + """Consume resources for a specific host. 'host' is a tuple + of the hostname and the services""" + + consume_func = getattr(self, '%s_consume' % topic, None) + if not consume_func: + return + consume_func(capabilities, instance_type) From e56d9409e11ee71cad8bc2d7eaf86c2c29e88117 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 06:52:19 -0700 Subject: [PATCH 052/107] minor fixups --- nova/scheduler/host_filter.py | 4 +--- nova/scheduler/zone_aware_scheduler.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 818ae4a30..b336665be 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -305,11 +305,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler): 'instance_type': } """ - def filter_hosts(self, topic, request_spec, hosts): + def filter_hosts(self, topic, request_spec, hosts=None): """Filter the full host list (from the ZoneManager)""" - if hosts: - return hosts filter_name = request_spec.get('filter', None) host_filter = choose_host_filter(filter_name) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index d4d3d0414..70cb83e8a 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -302,6 +302,7 @@ class ZoneAwareScheduler(driver.Scheduler): continue if filter_func(host, services['topic'], request_spec): filtered_hosts.append((host, services['topic'])) + return filtered_hosts def weigh_hosts(self, topic, request_spec, hosts): """Derived classes may override this to provide more sophisticated From 70e26a0b03bddcee03ce0b80c811ee4cbbf6b16b Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 09:43:27 -0700 Subject: [PATCH 053/107] an int() was missed being removed from UUID changes when zone rerouting kicks in --- nova/scheduler/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 1bb047e2e..0aed75680 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -169,7 +169,7 @@ def _issue_novaclient_command(nova, zone, collection, method_name, item_id): result = None try: try: - result = manager.get(int(item_id)) + result = manager.get(item_id) except ValueError, e: result = manager.find(name=item_id) except novaclient.NotFound: From d0aeb7c64225f14fe64319eb2caf7a83abe79429 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 13:54:45 -0400 Subject: [PATCH 054/107] Add admin api test case (like cloud test case) with a test for fw rules. --- nova/tests/test_adminapi.py | 90 +++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 nova/tests/test_adminapi.py diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py new file mode 100644 index 000000000..70a00f999 --- /dev/null +++ b/nova/tests/test_adminapi.py @@ -0,0 +1,90 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from eventlet import greenthread + +from nova import context +from nova import db +from nova import flags +from nova import log as logging +from nova import rpc +from nova import test +from nova import utils +from nova.auth import manager +from nova.api.ec2 import admin +from nova.image import fake + + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.tests.adminapi') + + +class AdminApiTestCase(test.TestCase): + def setUp(self): + super(AdminApiTestCase, self).setUp() + self.flags(connection_type='fake') + + self.conn = rpc.Connection.instance() + + # set up our cloud + self.api = admin.AdminController() + + # set up services + self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') + self.network = self.start_service('network') + self.volume = self.start_service('volume') + self.image_service = utils.import_object(FLAGS.image_service) + + self.manager = manager.AuthManager() + self.user = self.manager.create_user('admin', 'admin', 'admin', True) + self.project = self.manager.create_project('proj', 'admin', 'proj') + self.context = context.RequestContext(user=self.user, + project=self.project) + host = self.network.get_network_host(self.context.elevated()) + + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'available'}} + + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) + + # NOTE(vish): set up a manual wait so rpc.cast has a chance to finish + rpc_cast = rpc.cast + + def finish_cast(*args, **kwargs): + rpc_cast(*args, **kwargs) + greenthread.sleep(0.2) + + self.stubs.Set(rpc, 'cast', finish_cast) + + def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) + self.manager.delete_project(self.project) + self.manager.delete_user(self.user) + super(AdminApiTestCase, self).tearDown() + + def test_block_external_ips(self): + """Make sure provider firewall rules are created.""" + result = self.api.block_external_addresses(self.context, '1.1.1.1/32') + self.assertEqual('OK', result['status']) + self.assertEqual('Added 3 rules', result['message']) + From 1c8c0ba2589ca7af0a049f0d40393b66dc4744dd Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 13:59:26 -0400 Subject: [PATCH 055/107] pep8: remove newline at end of file. --- nova/tests/test_adminapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 70a00f999..7ecaf1c09 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -87,4 +87,3 @@ class AdminApiTestCase(test.TestCase): result = self.api.block_external_addresses(self.context, '1.1.1.1/32') self.assertEqual('OK', result['status']) self.assertEqual('Added 3 rules', result['message']) - From 63e96c261199c09a743874c8097bf5cca9001409 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 14:13:27 -0400 Subject: [PATCH 056/107] Add test for listing provider firewall rules. --- nova/tests/test_adminapi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 7ecaf1c09..2b90d49e9 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -87,3 +87,10 @@ class AdminApiTestCase(test.TestCase): result = self.api.block_external_addresses(self.context, '1.1.1.1/32') self.assertEqual('OK', result['status']) self.assertEqual('Added 3 rules', result['message']) + + def test_list_blocked_ips(self): + """Make sure we can see the external blocks that exist.""" + result = self.api.describe_external_address_blocks(self.context) + num = len(db.provider_fw_rule_get_all(self.context)) + # we only list IP, not tcp/udp/icmp rules + self.assertEqual(num / 3, len(result['externalIpBlockInfo'])) From 1387e757ff9bad6d09bc231878292ac038305775 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 14:16:11 -0400 Subject: [PATCH 057/107] Make sure there are actually rules to test against. --- nova/tests/test_adminapi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 2b90d49e9..4a96a3dd9 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -90,6 +90,7 @@ class AdminApiTestCase(test.TestCase): def test_list_blocked_ips(self): """Make sure we can see the external blocks that exist.""" + self.api.block_external_addresses(self.context, '1.1.1.2/32') result = self.api.describe_external_address_blocks(self.context) num = len(db.provider_fw_rule_get_all(self.context)) # we only list IP, not tcp/udp/icmp rules From 72b11602b2ad5e3fcc82464f507f6ac1970f936d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 14:45:37 -0400 Subject: [PATCH 058/107] Make firewall rules tests idempotent, move IPy=>netaddr, add deltete test. --- nova/tests/test_adminapi.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 4a96a3dd9..ce826fd5b 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -85,6 +85,7 @@ class AdminApiTestCase(test.TestCase): def test_block_external_ips(self): """Make sure provider firewall rules are created.""" result = self.api.block_external_addresses(self.context, '1.1.1.1/32') + self.api.remove_external_address_block(self.context, '1.1.1.1/32') self.assertEqual('OK', result['status']) self.assertEqual('Added 3 rules', result['message']) @@ -93,5 +94,18 @@ class AdminApiTestCase(test.TestCase): self.api.block_external_addresses(self.context, '1.1.1.2/32') result = self.api.describe_external_address_blocks(self.context) num = len(db.provider_fw_rule_get_all(self.context)) + self.api.remove_external_address_block(self.context, '1.1.1.2/32') # we only list IP, not tcp/udp/icmp rules self.assertEqual(num / 3, len(result['externalIpBlockInfo'])) + + def test_remove_ip_block(self): + """Remove ip blocks.""" + result = self.api.block_external_addresses(self.context, '1.1.1.3/32') + self.assertEqual('OK', result['status']) + num0 = len(db.provider_fw_rule_get_all(self.context)) + result = self.api.remove_external_address_block(self.context, + '1.1.1.3/32') + self.assertEqual('OK', result['status']) + self.assertEqual('Deleted 3 rules', result['message']) + num1 = len(db.provider_fw_rule_get_all(self.context)) + self.assert_(num1 < num0) From 64fdaf11b1a94622465c993b974eb745ecec87d6 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 23 Jun 2011 14:52:58 -0400 Subject: [PATCH 059/107] libvirt test for deleting provider firewall rules. --- nova/tests/test_libvirt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index ee94d3c17..d12e21063 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -1115,6 +1115,13 @@ class IptablesFirewallTestCase(test.TestCase): provjump_rules.append(rule) self.assertEqual(1, len(provjump_rules)) + # remove a rule from the db, cast to compute to refresh rule + db.provider_fw_rule_destroy(admin_ctxt, provider_fw1['id']) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(1, len(rules)) + class NWFilterTestCase(test.TestCase): def setUp(self): From c45dcfb0268f76a9748ef83bde6f79163ffbcd5a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Thu, 23 Jun 2011 21:31:00 -0400 Subject: [PATCH 060/107] Created Bootstrapper to handle Nova bootstrapping logic. --- bin/nova-api | 5 +++-- nova/service.py | 36 +----------------------------------- nova/utils.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 37 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 121f6f9a0..ea99a1b48 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -25,17 +25,18 @@ Starts both the EC2 and OpenStack APIs in separate processes. import sys -import nova.log import nova.service +import nova.utils def main(): """Launch EC2 and OSAPI services.""" - launcher = nova.service.Launcher(sys.argv) + nova.utils.Bootstrapper.bootstrap_binary(sys.argv) ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") + launcher = nova.service.Launcher() launcher.launch_service(ec2) launcher.launch_service(osapi) diff --git a/nova/service.py b/nova/service.py index 41c6551e0..00e4f61e5 100644 --- a/nova/service.py +++ b/nova/service.py @@ -60,47 +60,13 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", class Launcher(object): """Launch one or more services and wait for them to complete.""" - def __init__(self, flags=None): + def __init__(self): """Initialize the service launcher. - :param flags: Flags to use for the services we're going to load. :returns: None """ self._services = [] - self._version = version.version_string_with_vcs() - self._flags = flags - self._setup_flags() - self._setup_logging() - self._log_flags() - - def _setup_logging(self): - """Logic to ensure logging is going to work correctly for services. - - :returns: None - - """ - logging.setup() - logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) - - def _setup_flags(self): - """Logic to ensure flags/configuration are correctly set. - - :returns: None - - """ - utils.default_flagfile(args=self._flags) - FLAGS(self._flags or []) - flags.DEFINE_flag(flags.HelpFlag()) - flags.DEFINE_flag(flags.HelpshortFlag()) - flags.DEFINE_flag(flags.HelpXMLFlag()) - FLAGS.ParseNewFlags() - - def _log_flags(self): - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) @staticmethod def run_service(service): diff --git a/nova/utils.py b/nova/utils.py index a9b0f3128..a6b8d4cbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -743,3 +743,40 @@ def is_uuid_like(val): if not isinstance(val, basestring): return False return (len(val) == 36) and (val.count('-') == 4) + + +class Bootstrapper(object): + """Provides environment bootstrapping capabilities for entry points.""" + + @staticmethod + def bootstrap_binary(argv): + """Initialize the Nova environment using command line arguments.""" + Bootstrapper.setup_flags(argv) + Bootstrapper.setup_logging() + Bootstrapper.log_flags() + + @staticmethod + def setup_logging(): + """Initialize logging and log a message indicating the Nova version.""" + logging.setup() + logging.audit(_("Nova Version (%s)") % + version.version_string_with_vcs()) + + @staticmethod + def setup_flags(input_flags): + """Initialize flags, load flag file, and print help if needed.""" + default_flagfile(args=input_flags) + FLAGS(input_flags or []) + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() + + @staticmethod + def log_flags(): + """Log the list of all active flags being used.""" + logging.audit(_("Currently active flags:")) + for key in FLAGS: + value = FLAGS.get(key, None) + logging.audit(_("%(key)s : %(value)s" % locals())) + From 61263bb64abeb78b299f2b21127c110367573568 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 21:59:54 -0700 Subject: [PATCH 061/107] missed passing an argument to consume_resources --- nova/scheduler/zone_aware_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 70cb83e8a..073bdd3bd 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -250,7 +250,7 @@ class ZoneAwareScheduler(driver.Scheduler): weights.sort(key=operator.itemgetter('weight')) best_weight = weights[0] weighted.append(best_weight) - self.consume_resources(best_weight['capabilities'], + self.consume_resources(topic, best_weight['capabilities'], instance_type) # Next, tack on the best weights from the child zones ... From 26caab74aeb45a1bc759e9b44f28c84853b21ce8 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 01:34:47 -0400 Subject: [PATCH 062/107] pep8 fixes --- nova/scheduler/host_filter.py | 24 ++++++++++++++++++++++-- nova/tests/scheduler/test_host_filter.py | 24 +++++++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index bd6b26608..7d4b9ad91 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -93,6 +93,22 @@ class InstanceTypeFilter(HostFilter): """Use instance_type to filter hosts.""" return (self._full_name(), instance_type) + def _satisfies_extra_specs(self, capabilities, instance_type): + """Check that the capabilities provided by the compute service + satisfy the extra specs associated with the instance type""" + + # Note(lorinh): For now, we are just checking exact matching on the + # values. Later on, we want to handle numerical + # values so we can represent things like number of GPU cards + try: + for key, value in instance_type['extra_specs'].iteritems(): + if capabilities[key] != value: + return False + except KeyError: + return False + + return True + def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" instance_type = query @@ -103,8 +119,12 @@ class InstanceTypeFilter(HostFilter): disk_bytes = capabilities['disk_available'] spec_ram = instance_type['memory_mb'] spec_disk = instance_type['local_gb'] - if host_ram_mb >= spec_ram and disk_bytes >= spec_disk: - selected_hosts.append((host, capabilities)) + extra_specs = instance_type['extra_specs'] + + if host_ram_mb >= spec_ram and \ + disk_bytes >= spec_disk and \ + self._satisfies_extra_specs(capabilities, instance_type): + selected_hosts.append((host, capabilities)) return selected_hosts #host entries (currently) are like: diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 10eafde08..75a2cb21c 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -67,13 +67,24 @@ class HostFilterTestCase(test.TestCase): flavorid=1, swap=500, rxtx_quota=30000, - rxtx_cap=200) + rxtx_cap=200, + extra_specs={}) + self.gpu_instance_type = dict(name='tiny.gpu', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=2, + swap=500, + rxtx_quota=30000, + rxtx_cap=200, + extra_specs={'gpu': 'nvidia'}) self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states + self.zone_manager.service_states['host07']['compute']['gpu'] = 'nvidia' def tearDown(self): FLAGS.default_host_filter = self.old_flag @@ -116,6 +127,17 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host05', just_hosts[0]) self.assertEquals('host10', just_hosts[5]) + def test_instance_type_filter_extra_specs(self): + hf = host_filter.InstanceTypeFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = hf.instance_type_to_filter(self.gpu_instance_type) + self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', + name) + hosts = hf.filter_hosts(self.zone_manager, cooked) + self.assertEquals(1, len(hosts)) + just_hosts = [host for host, caps in hosts] + self.assertEquals('host07', just_hosts[0]) + def test_json_filter(self): hf = host_filter.JsonFilter() # filter all hosts that can support 50 ram and 500 disk From eb3af324e247faf240a769dc6b01896cdab763d6 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 01:44:22 -0400 Subject: [PATCH 063/107] Dealing with cases where extra_specs wasn't defined --- nova/scheduler/host_filter.py | 4 ++++ nova/tests/test_host_filter.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 7d4b9ad91..9c396cc0c 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -97,9 +97,13 @@ class InstanceTypeFilter(HostFilter): """Check that the capabilities provided by the compute service satisfy the extra specs associated with the instance type""" + if 'extra_specs' not in instance_type: + return True + # Note(lorinh): For now, we are just checking exact matching on the # values. Later on, we want to handle numerical # values so we can represent things like number of GPU cards + try: for key, value in instance_type['extra_specs'].iteritems(): if capabilities[key] != value: diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index 3361c7b73..438f3e522 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -67,7 +67,8 @@ class HostFilterTestCase(test.TestCase): flavorid=1, swap=500, rxtx_quota=30000, - rxtx_cap=200) + rxtx_cap=200, + extra_specs={}) self.zone_manager = FakeZoneManager() states = {} From 2e9aa262764a5d72bc688f6e4397cb82635b2496 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 22:48:44 -0700 Subject: [PATCH 064/107] debug logging of number of instances to build in scheduler --- nova/scheduler/zone_aware_scheduler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 073bdd3bd..a747526d1 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -180,18 +180,21 @@ class ZoneAwareScheduler(driver.Scheduler): request_spec, kwargs) return None + num_instances = request_spec['num_instances'] + LOG.debug(_("Attemping to build %d instance%s") % + (num_instances, "" if num_instances == 1 else "s")) + # Create build plan and provision ... build_plan = self.select(context, request_spec) if not build_plan: raise driver.NoValidHost(_('No hosts were available')) - for num in xrange(request_spec['num_instances']): + for num in xrange(num_instances): if not build_plan: break - item = build_plan.pop(0) - self._provision_resource(context, item, instance_id, request_spec, - kwargs) + self._provision_resource(context, item, instance_id, + request_spec, kwargs) # Returning None short-circuits the routing to Compute (since # we've already done it here) From 564d0094ff29f62299a52da9bd704f0a155b000a Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 09:54:38 +0400 Subject: [PATCH 065/107] Unwind last commit, force anyjson to use our serialization methods. --- bin/nova-manage | 8 ++++---- nova/utils.py | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 0147ae21b..e09ea495d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -874,10 +874,10 @@ class InstanceTypeCommands(object): try: instance_types.create(name, memory, vcpus, local_gb, flavorid, swap, rxtx_quota, rxtx_cap) - except exception.InvalidInputException: - print "Must supply valid parameters to create instance_type" - print e - sys.exit(1) + #except exception.InvalidInputException: + # print "Must supply valid parameters to create instance_type" + # print e + # sys.exit(1) except exception.ApiError, e: print "\n\n" print "\n%s" % e diff --git a/nova/utils.py b/nova/utils.py index 691134ada..a77cf7993 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -525,6 +525,15 @@ def loads(s): return json.loads(s) +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append(("nova.utils", "dumps", TypeError, "loads", ValueError)) + anyjson.force_implementation("nova.utils") + + _semaphores = {} From a862f1f69666aebc31db35f13906a02854527c1a Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 22:55:45 -0700 Subject: [PATCH 066/107] typo in least cost scheduler --- nova/scheduler/least_cost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py index 72db2fd1b..9376631ef 100644 --- a/nova/scheduler/least_cost.py +++ b/nova/scheduler/least_cost.py @@ -63,7 +63,7 @@ def compute_fill_first_cost_fn(host): class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler): def __init__(self, *args, **kwargs): self.cost_fns_cache = {} - super(LeastCoastScheduler, self).__init__(*args, **kwargs) + super(LeastCostScheduler, self).__init__(*args, **kwargs) def get_cost_fns(self, topic): """Returns a list of tuples containing weights and cost functions to From 61a299d9824faaa6b819e8092cdc7f555269c97f Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 23:00:15 -0700 Subject: [PATCH 067/107] more typos --- nova/scheduler/zone_aware_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index a747526d1..8f218c159 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -303,8 +303,8 @@ class ZoneAwareScheduler(driver.Scheduler): for host, services in host_list: if topic not in services: continue - if filter_func(host, services['topic'], request_spec): - filtered_hosts.append((host, services['topic'])) + if filter_func(host, services[topic], request_spec): + filtered_hosts.append((host, services[topic])) return filtered_hosts def weigh_hosts(self, topic, request_spec, hosts): From 0f5526b2235378348169bfbc947d330dd3b3bd0e Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 23:05:12 -0700 Subject: [PATCH 068/107] requested_mem typo --- nova/scheduler/zone_aware_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 8f218c159..769b2dd0f 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -279,7 +279,7 @@ class ZoneAwareScheduler(driver.Scheduler): is acceptable for scheduling. """ instance_type = request_spec['instance_type'] - reqested_mem = instance_type['memory_mb'] + requested_mem = instance_type['memory_mb'] return capabilities['host_memory_free'] >= requested_mem def filter_hosts(self, topic, request_spec, host_list=None): From c9d58da89423cacccc1b9db1a9ec3d66f42c1d4d Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 23 Jun 2011 23:38:32 -0700 Subject: [PATCH 069/107] LeastCostScheduler wasn't checking for topic cost functions correctly. Added support so that --least_cost_scheduler_cost_functions only needs to have method names specified, instead of the full blown version with module and class name. Still works the old way, too. --- nova/scheduler/least_cost.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py index 9376631ef..6f5eb66fd 100644 --- a/nova/scheduler/least_cost.py +++ b/nova/scheduler/least_cost.py @@ -75,8 +75,15 @@ class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler): cost_fns = [] for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions: - if not cost_fn_str.startswith('%s_' % topic) and \ - not cost_fn_str.startswith('noop'): + if '.' in cost_fn_str: + short_name = cost_fn_str.split('.')[-1] + else: + short_name = cost_fn_str + cost_fn_str = "%s.%s.%s" % ( + __name__, self.__class__.__name__, short_name) + + if not (short_name.startswith('%s_' % topic) or + short_name.startswith('noop')): continue try: From 9536099739012fef88dfd436ef67d97ba56346a3 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 24 Jun 2011 00:26:55 -0700 Subject: [PATCH 070/107] on 2nd run through filter_hosts, we've already accounted for the topic memory needs converted to Bytes from MB --- nova/scheduler/zone_aware_scheduler.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 769b2dd0f..e6383b20b 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -244,6 +244,8 @@ class ZoneAwareScheduler(driver.Scheduler): # may have been consumed from a previous build.. host_list = self.filter_hosts(topic, request_spec, host_list) if not host_list: + LOG.warn(_("Ran out of available hosts after weighing " + "%d of %d instances") % (i, num_instances)) break # then weigh the selected hosts. @@ -279,7 +281,7 @@ class ZoneAwareScheduler(driver.Scheduler): is acceptable for scheduling. """ instance_type = request_spec['instance_type'] - requested_mem = instance_type['memory_mb'] + requested_mem = instance_type['memory_mb'] * 1024 * 1024 return capabilities['host_memory_free'] >= requested_mem def filter_hosts(self, topic, request_spec, host_list=None): @@ -297,14 +299,20 @@ class ZoneAwareScheduler(driver.Scheduler): filter_func = getattr(self, '%s_filter' % topic, _default_filter) - filtered_hosts = [] if host_list is None: + first_run = True host_list = self.zone_manager.service_states.iteritems() + else: + first_run = False + + filtered_hosts = [] for host, services in host_list: - if topic not in services: - continue - if filter_func(host, services[topic], request_spec): - filtered_hosts.append((host, services[topic])) + if first_run: + if topic not in services: + continue + services = services['topic'] + if filter_func(host, services, request_spec): + filtered_hosts.append((host, services)) return filtered_hosts def weigh_hosts(self, topic, request_spec, hosts): @@ -318,7 +326,7 @@ class ZoneAwareScheduler(driver.Scheduler): def compute_consume(self, capabilities, instance_type): """Consume compute resources for selected host""" - requested_mem = max(instance_type['memory_mb'], 0) + requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024 capabilities['host_memory_free'] -= requested_mem def consume_resources(self, topic, capabilities, instance_type): From e3e435f3a4cf6ab1a75551c6298587efabb6eabd Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 24 Jun 2011 00:30:58 -0700 Subject: [PATCH 071/107] same typo i made before! --- nova/scheduler/zone_aware_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index e6383b20b..e24c2256f 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -310,7 +310,7 @@ class ZoneAwareScheduler(driver.Scheduler): if first_run: if topic not in services: continue - services = services['topic'] + services = services[topic] if filter_func(host, services, request_spec): filtered_hosts.append((host, services)) return filtered_hosts From 61bbfacf321569f9fa624ffbc86c09fc19df8eca Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:20:24 +0400 Subject: [PATCH 072/107] Add reconnect on server fail to LDAP driver. --- nova/auth/ldapdriver.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index e9532473d..4af91b613 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -101,6 +101,41 @@ def sanitize(fn): return _wrapped +class LDAPWrapper(object): + def __init__(self, ldap, url, user, password): + self.ldap = ldap + self.url = url + self.user = user + self.password = password + self.conn = None + + def __wrap_reconnect(f): + def inner(self, *args, **kwargs): + if self.conn is None: + self.connect() + return f(self.conn)(*args, **kwargs) + else: + try: + return f(self.conn)(*args, **kwargs) + except self.ldap.SERVER_DOWN: + self.connect() + return f(self.conn)(*args, **kwargs) + return inner + + def connect(self): + try: + self.conn = self.ldap.initialize(self.url) + self.conn.bind_s(self.user, self.password) + except self.ldap.SERVER_DOWN: + self.conn = None + raise + + search_s = __wrap_reconnect(lambda conn: conn.search_s) + add_s = __wrap_reconnect(lambda conn: conn.add_s) + delete_s = __wrap_reconnect(lambda conn: conn.delete_s) + modify_s = __wrap_reconnect(lambda conn: conn.modify_s) + + class LdapDriver(object): """Ldap Auth driver @@ -124,8 +159,8 @@ class LdapDriver(object): LdapDriver.project_objectclass = 'novaProject' self.__cache = None if LdapDriver.conn is None: - LdapDriver.conn = self.ldap.initialize(FLAGS.ldap_url) - LdapDriver.conn.simple_bind_s(FLAGS.ldap_user_dn, + LdapDriver.conn = LDAPWrapper(self.ldap, FLAGS.ldap_url, + FLAGS.ldap_user_dn, FLAGS.ldap_password) if LdapDriver.mc is None: LdapDriver.mc = memcache.Client(FLAGS.memcached_servers, debug=0) From d7bf56edff9620c6554ab918046d4b3aac7b9b04 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:26:15 +0400 Subject: [PATCH 073/107] Use simple_bind_s instead of bind_s --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 4af91b613..bc37d2d87 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -125,7 +125,7 @@ class LDAPWrapper(object): def connect(self): try: self.conn = self.ldap.initialize(self.url) - self.conn.bind_s(self.user, self.password) + self.conn.simple_bind_s(self.user, self.password) except self.ldap.SERVER_DOWN: self.conn = None raise From 007370322fb9fa023cd0326dd6ced3bdcb264027 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:55:06 +0400 Subject: [PATCH 074/107] Add reconnect test. --- nova/auth/fakeldap.py | 24 ++++++++++++++++++++++++ nova/tests/test_auth.py | 9 +++++++++ 2 files changed, 33 insertions(+) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 79afb9109..f1e769278 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -100,6 +100,11 @@ class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable=C0103 pass +class SERVER_DOWN(Exception): # pylint: disable=C0103 + """Duplicate exception class from real LDAP module.""" + pass + + def initialize(_uri): """Opens a fake connection with an LDAP server.""" return FakeLDAP() @@ -202,25 +207,38 @@ def _to_json(unencoded): return json.dumps(list(unencoded)) +server_fail = False + + class FakeLDAP(object): """Fake LDAP connection.""" def simple_bind_s(self, dn, password): """This method is ignored, but provided for compatibility.""" + if server_fail: + raise SERVER_DOWN pass def unbind_s(self): """This method is ignored, but provided for compatibility.""" + if server_fail: + raise SERVER_DOWN pass def add_s(self, dn, attr): """Add an object with the specified attributes at dn.""" + if server_fail: + raise SERVER_DOWN + key = "%s%s" % (self.__prefix, dn) value_dict = dict([(k, _to_json(v)) for k, v in attr]) Store.instance().hmset(key, value_dict) def delete_s(self, dn): """Remove the ldap object at specified dn.""" + if server_fail: + raise SERVER_DOWN + Store.instance().delete("%s%s" % (self.__prefix, dn)) def modify_s(self, dn, attrs): @@ -232,6 +250,9 @@ class FakeLDAP(object): ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value) """ + if server_fail: + raise SERVER_DOWN + store = Store.instance() key = "%s%s" % (self.__prefix, dn) @@ -255,6 +276,9 @@ class FakeLDAP(object): fields -- fields to return. Returns all fields if not specified """ + if server_fail: + raise SERVER_DOWN + if scope != SCOPE_BASE and scope != SCOPE_SUBTREE: raise NotImplementedError(str(scope)) store = Store.instance() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 7d00bddfe..4aebbe940 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,6 +25,7 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud +from nova.auth import fakeldap FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -369,6 +370,14 @@ class _AuthManagerBaseTestCase(test.TestCase): class AuthManagerLdapTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' + def test_reconnect_on_server_failure(self): + self.manager.get_users() + fakeldap.server_fail = True + with self.assertRaises(fakeldap.SERVER_DOWN): + self.manager.get_users() + fakeldap.server_fail = False + self.manager.get_users() + class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' From 26cc4e4fd04432586cd438f3e5205cfd15e5ccb4 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 10:52:59 -0400 Subject: [PATCH 075/107] Edited the host filter test case for extra specs --- nova/scheduler/host_filter.py | 4 ++-- nova/tests/scheduler/test_host_filter.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 9c396cc0c..5f61f9456 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -103,7 +103,7 @@ class InstanceTypeFilter(HostFilter): # Note(lorinh): For now, we are just checking exact matching on the # values. Later on, we want to handle numerical # values so we can represent things like number of GPU cards - + try: for key, value in instance_type['extra_specs'].iteritems(): if capabilities[key] != value: @@ -128,7 +128,7 @@ class InstanceTypeFilter(HostFilter): if host_ram_mb >= spec_ram and \ disk_bytes >= spec_disk and \ self._satisfies_extra_specs(capabilities, instance_type): - selected_hosts.append((host, capabilities)) + selected_hosts.append((host, capabilities)) return selected_hosts #host entries (currently) are like: diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 75a2cb21c..b1892dab4 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -77,14 +77,26 @@ class HostFilterTestCase(test.TestCase): swap=500, rxtx_quota=30000, rxtx_cap=200, - extra_specs={'gpu': 'nvidia'}) + extra_specs={'xpu_arch': 'fermi', + 'xpu_info': 'Tesla 2050'}) self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states - self.zone_manager.service_states['host07']['compute']['gpu'] = 'nvidia' + + # Add some extra capabilities to some hosts + host07 = self.zone_manager.service_states['host07']['compute'] + host07['xpu_arch'] = 'fermi' + host07['xpu_info'] = 'Tesla 2050' + + host08 = self.zone_manager.service_states['host08']['compute'] + host08['xpu_arch'] = 'radeon' + + host09 = self.zone_manager.service_states['host09']['compute'] + host09['xpu_arch'] = 'fermi' + host09['xpu_info'] = 'Tesla 2150' def tearDown(self): FLAGS.default_host_filter = self.old_flag From 9be3dbf53431f94501f4625b80f971458843a0aa Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Fri, 24 Jun 2011 23:55:18 +0400 Subject: [PATCH 077/107] stub tests --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index 94fcf7f5b..bf2ba81d9 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From dd72dbbc85e97e283d331083cf4ae43582bca4f6 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 24 Jun 2011 15:03:01 -0500 Subject: [PATCH 078/107] Refactored backup rotate. --- nova/exception.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/exception.py b/nova/exception.py index a548a638c..f3893d239 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -553,6 +553,10 @@ class ImageRotationNotAllowed(NovaException): message = _("Rotation is not allowed for snapshots") +class RotationRequiredForBackup(NovaException): + message = _("Rotation param is required for backup image_type") + + #TODO(bcwaldon): EOL this exception! class Duplicate(NovaException): pass From e2687d065dc172dbe67dbf11f7af23d15119a26c Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 00:33:40 +0400 Subject: [PATCH 079/107] some tests --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index bf2ba81d9..09064051d 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From 33573b53ec77ed32dad2cb04579323f7e10cc743 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 03:03:17 +0400 Subject: [PATCH 080/107] fixes --- Authors | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Authors b/Authors index 09064051d..c4281ca16 100644 --- a/Authors +++ b/Authors @@ -22,14 +22,14 @@ David Pravec Dean Troyer Devin Carlen Ed Leafe -Eldar Nugaev +Eldar Nugaev Eric Day Eric Windisch Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker @@ -53,6 +53,7 @@ Kei Masumoto Ken Pepple Kevin Bringard Kevin L. Mitchell +Kirill Shileev Koji Iida Lorin Hochstein Lvov Maxim From 078b9b473a6d727b93539bae818134fc9d251948 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Sat, 25 Jun 2011 03:05:09 +0400 Subject: [PATCH 081/107] added disassociate method to tests --- nova/exception.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/exception.py b/nova/exception.py index 64abff9af..8230c2b26 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -359,9 +359,11 @@ class DatastoreNotFound(NotFound): class NoFixedIpsFoundForInstance(NotFound): message = _("Instance %(instance_id)s has zero fixed ips.") + class FloatingIpNotDefined(NotFound): message = _("Floating ip %(floating_ip)s not found") + class FloatingIpNotFound(NotFound): message = _("Floating ip not found for fixed address %(fixed_ip)s.") From d5891f620bd8a117e57c70255f74239f0ce19600 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 04:45:15 +0400 Subject: [PATCH 082/107] mailmap --- .mailmap | 2 ++ Authors | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 3f0238ee9..89094fbbb 100644 --- a/.mailmap +++ b/.mailmap @@ -47,3 +47,5 @@ + + diff --git a/Authors b/Authors index c4281ca16..21a9bf94b 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From 4238d0aad0cf4704c8092f7b375f651dea615713 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 04:47:06 +0400 Subject: [PATCH 083/107] mailmap --- .mailmap | 2 ++ Authors | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index 89094fbbb..6673d0a26 100644 --- a/.mailmap +++ b/.mailmap @@ -49,3 +49,5 @@ + + \ No newline at end of file diff --git a/Authors b/Authors index 21a9bf94b..c3a65f1b4 100644 --- a/Authors +++ b/Authors @@ -22,7 +22,7 @@ David Pravec Dean Troyer Devin Carlen Ed Leafe -Eldar Nugaev +Eldar Nugaev Eric Day Eric Windisch Ewan Mellor @@ -53,7 +53,7 @@ Kei Masumoto Ken Pepple Kevin Bringard Kevin L. Mitchell -Kirill Shileev +Kirill Shileev Koji Iida Lorin Hochstein Lvov Maxim From 99381b313552368d752e3ec934ca4b51b881872e Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Sat, 25 Jun 2011 14:04:40 +0400 Subject: [PATCH 084/107] PEP8 fix --- nova/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index 8f9ca42c1..6d8324e5b 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -531,7 +531,8 @@ try: except ImportError: pass else: - anyjson._modules.append(("nova.utils", "dumps", TypeError, "loads", ValueError)) + anyjson._modules.append(("nova.utils", "dumps", TypeError, + "loads", ValueError)) anyjson.force_implementation("nova.utils") From 80a0df6ff094c09bedc24942a6303262659e8811 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 25 Jun 2011 17:26:38 -0700 Subject: [PATCH 085/107] only create the db if it doesn't exist, add an option -r to run_tests.py to delete it --- nova/tests/__init__.py | 2 +- run_tests.py | 7 +++---- run_tests.sh | 7 +++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 7fba02a93..5e0cb718e 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -50,7 +50,7 @@ def setup(): testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db) if os.path.exists(testdb): - os.unlink(testdb) + return migration.db_sync() ctxt = context.get_admin_context() network_manager.VlanManager().create_networks(ctxt, diff --git a/run_tests.py b/run_tests.py index bb33f9139..fd836967e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -69,7 +69,6 @@ from nose import core from nose import result from nova import log as logging -from nova.tests import fake_flags class _AnsiColorizer(object): @@ -211,11 +210,11 @@ class NovaTestResult(result.TextTestResult): break sys.stdout = stdout - # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate - # error results in it failing to be initialized later. Otherwise, + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, # _handleElapsedTime will fail, causing the wrong error message to # be outputted. - self.start_time = time.time() + self.start_time = time.time() def getDescription(self, test): return str(test) diff --git a/run_tests.sh b/run_tests.sh index c3f06f837..2ea221ae3 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,6 +6,7 @@ function usage { echo "" echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -r, --recreate-db Recreate the test database." echo " -x, --stop Stop running tests after the first error or failure." echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" @@ -23,6 +24,7 @@ function process_option { -h|--help) usage;; -V|--virtual-env) let always_venv=1; let never_venv=0;; -N|--no-virtual-env) let always_venv=0; let never_venv=1;; + -r|--recreate-db) let recreate_db=1;; -f|--force) let force=1;; -p|--pep8) let just_pep8=1;; -*) noseopts="$noseopts $1";; @@ -39,6 +41,7 @@ noseargs= noseopts= wrapper="" just_pep8=0 +recreate_db=0 for arg in "$@"; do process_option $arg @@ -108,6 +111,10 @@ if [ $just_pep8 -eq 1 ]; then exit fi +if [ $recreate_db -eq 1 ]; then + rm tests.sqlite +fi + run_tests || exit # NOTE(sirp): we only want to run pep8 when we're running the full-test suite, From 1af9f7ba4eda5cc9ec327cfc12ac1a5c72e2bbf4 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Mon, 27 Jun 2011 16:33:01 +0400 Subject: [PATCH 086/107] review issues fixed --- nova/exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 8230c2b26..8a63a79f2 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -360,11 +360,11 @@ class NoFixedIpsFoundForInstance(NotFound): message = _("Instance %(instance_id)s has zero fixed ips.") -class FloatingIpNotDefined(NotFound): +class FloatingIpNotFound(NotFound): message = _("Floating ip %(floating_ip)s not found") -class FloatingIpNotFound(NotFound): +class FloatingIpNotFoundForFixedAddress(NotFound): message = _("Floating ip not found for fixed address %(fixed_ip)s.") From 1adee45f0d84704c6e5dce46694c41cc9bffba53 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 27 Jun 2011 15:49:24 -0400 Subject: [PATCH 087/107] Added nova.version to utils.py --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/utils.py b/nova/utils.py index a6b8d4cbe..badd67599 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -46,6 +46,7 @@ from eventlet.green import subprocess from nova import exception from nova import flags from nova import log as logging +from nova import version LOG = logging.getLogger("nova.utils") From 8635818349c7f47e012c68fc1056d0f02a4fda25 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 27 Jun 2011 15:05:37 -0700 Subject: [PATCH 088/107] Made _issue_novaclient_command() behave better. Fixed a bunch of tests. --- nova/scheduler/api.py | 49 +++++++++++++------ nova/scheduler/zone_aware_scheduler.py | 4 +- .../scheduler/test_least_cost_scheduler.py | 11 +++-- nova/tests/scheduler/test_scheduler.py | 4 +- .../scheduler/test_zone_aware_scheduler.py | 35 ++++++------- 5 files changed, 60 insertions(+), 43 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 0aed75680..5d62beb86 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -162,32 +162,53 @@ def child_zone_helper(zone_list, func): _wrap_method(_process, func), zone_list)] -def _issue_novaclient_command(nova, zone, collection, method_name, item_id): +def _issue_novaclient_command(nova, zone, collection, + method_name, *args, **kwargs): """Use novaclient to issue command to a single child zone. - One of these will be run in parallel for each child zone.""" + One of these will be run in parallel for each child zone. + """ manager = getattr(nova, collection) - result = None - try: + + # NOTE(comstud): This is not ideal, but we have to do this based on + # how novaclient is implemented right now. + # 'find' is special cased as novaclient requires kwargs for it to + # filter on a 'get_all'. + # Every other method first needs to do a 'get' on the first argument + # passed, which should be a UUID. If it's 'get' itself that we want, + # we just return the result. Otherwise, we next call the real method + # that's wanted... passing other arguments that may or may not exist. + if method_name in ['find', 'findall']: try: - result = manager.get(item_id) - except ValueError, e: - result = manager.find(name=item_id) + return getattr(manager, method_name)(**kwargs) + except novaclient.NotFound: + url = zone.api_url + LOG.debug(_("%(collection)s.%(method_name)s didn't find " + "anything matching '%(kwargs)s' on '%(url)s'" % + locals())) + return None + + args = list(args) + # pop off the UUID to look up + item = args.pop(0) + try: + result = manager.get(item) except novaclient.NotFound: url = zone.api_url - LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % + LOG.debug(_("%(collection)s '%(item)s' not found on '%(url)s'" % locals())) return None - if method_name.lower() not in ['get', 'find']: - result = getattr(result, method_name)() + if method_name.lower() != 'get': + # if we're doing something other than 'get', call it passing args. + result = getattr(result, method_name)(*args, **kwargs) return result -def wrap_novaclient_function(f, collection, method_name, item_id): - """Appends collection, method_name and item_id to the incoming +def wrap_novaclient_function(f, collection, method_name, *args, **kwargs): + """Appends collection, method_name and arguments to the incoming (nova, zone) call from child_zone_helper.""" def inner(nova, zone): - return f(nova, zone, collection, method_name, item_id) + return f(nova, zone, collection, method_name, *args, **kwargs) return inner @@ -220,7 +241,7 @@ class reroute_compute(object): the wrapped method. (This ensures that zone-local code can continue to use integer IDs). - 4. If the item was not found, we delgate the call to a child zone + 4. If the item was not found, we delegate the call to a child zone using the UUID. """ def __init__(self, method_name): diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index e24c2256f..eb116fac4 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -180,7 +180,7 @@ class ZoneAwareScheduler(driver.Scheduler): request_spec, kwargs) return None - num_instances = request_spec['num_instances'] + num_instances = request_spec.get('num_instances', 1) LOG.debug(_("Attemping to build %d instance%s") % (num_instances, "" if num_instances == 1 else "s")) @@ -227,7 +227,7 @@ class ZoneAwareScheduler(driver.Scheduler): raise NotImplemented(_("Zone Aware Scheduler only understands " "Compute nodes (for now)")) - num_instances = request_spec['num_instances'] + num_instances = request_spec.get('num_instances', 1) instance_type = request_spec['instance_type'] weighted = [] diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py index 9a5318aee..49791053e 100644 --- a/nova/tests/scheduler/test_least_cost_scheduler.py +++ b/nova/tests/scheduler/test_least_cost_scheduler.py @@ -122,15 +122,16 @@ class LeastCostSchedulerTestCase(test.TestCase): for hostname, caps in hosts] self.assertWeights(expected, num, request_spec, hosts) - def test_fill_first_cost_fn(self): + def test_compute_fill_first_cost_fn(self): FLAGS.least_cost_scheduler_cost_functions = [ - 'nova.scheduler.least_cost.fill_first_cost_fn', + 'nova.scheduler.least_cost.compute_fill_first_cost_fn', ] - FLAGS.fill_first_cost_fn_weight = 1 + FLAGS.compute_fill_first_cost_fn_weight = 1 num = 1 - request_spec = {} - hosts = self.sched.filter_hosts(num, request_spec) + instance_type = {'memory_mb': 1024} + request_spec = {'instance_type': instance_type} + hosts = self.sched.filter_hosts('compute', request_spec, None) expected = [] for idx, (hostname, caps) in enumerate(hosts): diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index 4be59d411..fea8b424d 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -1074,7 +1074,7 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), - zone, "servers", "find", "name").b, 22) + zone, "servers", "find", name="test").b, 22) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), @@ -1088,7 +1088,7 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "find", "name"), None) + zone, "servers", "find", name="test"), None) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 37c6488cc..b2599f1b8 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -55,29 +55,21 @@ def fake_zone_manager_service_states(num_hosts): class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler): - def filter_hosts(self, num, specs): - # NOTE(sirp): this is returning [(hostname, services)] - return self.zone_manager.service_states.items() - - def weigh_hosts(self, num, specs, hosts): - fake_weight = 99 - weighted = [] - for hostname, caps in hosts: - weighted.append(dict(weight=fake_weight, name=hostname)) - return weighted + # No need to stub anything at the moment + pass class FakeZoneManager(zone_manager.ZoneManager): def __init__(self): self.service_states = { 'host1': { - 'compute': {'ram': 1000}, + 'compute': {'host_memory_free': 1000*1024*1024}, }, 'host2': { - 'compute': {'ram': 2000}, + 'compute': {'host_memory_free': 2000*1024*1024}, }, 'host3': { - 'compute': {'ram': 3000}, + 'compute': {'host_memory_free': 3000*1024*1024}, }, } @@ -164,13 +156,17 @@ class ZoneAwareSchedulerTestCase(test.TestCase): sched.set_zone_manager(zm) fake_context = {} - build_plan = sched.select(fake_context, {}) + build_plan = sched.select(fake_context, + {'instance_type': {'memory_mb': 512}, + 'num_instances': 4 }) - self.assertEqual(15, len(build_plan)) + # 4 from local zones, 12 from remotes + self.assertEqual(16, len(build_plan)) - hostnames = [plan_item['name'] - for plan_item in build_plan if 'name' in plan_item] - self.assertEqual(3, len(hostnames)) + hostnames = [plan_item['hostname'] + for plan_item in build_plan if 'hostname' in plan_item] + # 4 local hosts + self.assertEqual(4, len(hostnames)) def test_empty_zone_aware_scheduler(self): """ @@ -185,8 +181,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase): fake_context = {} self.assertRaises(driver.NoValidHost, sched.schedule_run_instance, fake_context, 1, - dict(host_filter=None, - request_spec={'instance_type': {}})) + dict(host_filter=None, instance_type={})) def test_schedule_do_not_schedule_with_hint(self): """ From 7bb2fde277bdcdc96ef3f9df10ab21ebcd7caf09 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 27 Jun 2011 15:17:19 -0700 Subject: [PATCH 089/107] logging fixes --- nova/scheduler/zone_aware_scheduler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index eb116fac4..638072ee7 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -181,8 +181,7 @@ class ZoneAwareScheduler(driver.Scheduler): return None num_instances = request_spec.get('num_instances', 1) - LOG.debug(_("Attemping to build %d instance%s") % - (num_instances, "" if num_instances == 1 else "s")) + LOG.debug(_("Attemping to build %d instance(s)") % locals()) # Create build plan and provision ... build_plan = self.select(context, request_spec) @@ -245,7 +244,7 @@ class ZoneAwareScheduler(driver.Scheduler): host_list = self.filter_hosts(topic, request_spec, host_list) if not host_list: LOG.warn(_("Ran out of available hosts after weighing " - "%d of %d instances") % (i, num_instances)) + "%(i)d of %(num_instances)d instances") % locals()) break # then weigh the selected hosts. From 67a1db663fda19cf1010b2b4eb2e97810f1f54a4 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 28 Jun 2011 08:57:05 +0000 Subject: [PATCH 091/107] Re-merging code for generating system-usages to get around bzr merge braindeadness. --- bin/instance-usage-audit | 127 +++++++++++++++++++++++++++++++++ nova/notifier/test_notifier.py | 28 ++++++++ nova/tests/test_compute.py | 77 ++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100755 bin/instance-usage-audit create mode 100644 nova/notifier/test_notifier.py diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit new file mode 100755 index 000000000..1124cf550 --- /dev/null +++ b/bin/instance-usage-audit @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Cron script to generate usage notifications for instances neither created + nor destroyed in a given time period. + + Together with the notifications generated by compute on instance + create/delete/resize, over that ime period, this allows an external + system consuming usage notification feeds to calculate instance usage + for each tenant. + + Time periods are specified like so: + [mdy] + + 1m = previous month. If the script is run April 1, it will generate usages + for March 1 thry March 31. + 3m = 3 previous months. + 90d = previous 90 days. + 1y = previous year. If run on Jan 1, it generates usages for + Jan 1 thru Dec 31 of the previous year. +""" + +import datetime +import gettext +import os +import sys +import time + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')): + sys.path.insert(0, POSSIBLE_TOPDIR) + +gettext.install('nova', unicode=1) + + +from nova import context +from nova import db +from nova import exception +from nova import flags +from nova import log as logging +from nova import utils + +from nova.notifier import api as notifier_api + +FLAGS = flags.FLAGS +flags.DEFINE_string('instance_usage_audit_period', '1m', + 'time period to generate instance usages for.') + +def time_period(period): + today = datetime.date.today() + unit = period[-1] + if unit not in 'mdy': + raise ValueError('Time period must be m, d, or y') + n = int(period[:-1]) + if unit == 'm': + year = today.year - (n//12) + n = n % 12 + if n >= today.month: + year -= 1 + month = 12 + (today.month - n) + else: + month = today.month - n + begin = datetime.datetime(day=1, month=month, year=year) + end = datetime.datetime(day=1, month=today.month, year=today.year) + + elif unit == 'y': + begin = datetime.datetime(day=1, month=1, year=today.year - n) + end = datetime.datetime(day=1, month=1, year=today.year) + + elif unit == 'd': + b = today - datetime.timedelta(days=n) + begin = datetime.datetime(day=b.day, month=b.month, year=b.year) + end = datetime.datetime(day=today.day, + month=today.month, + year=today.year) + + return (begin, end) + +if __name__ == '__main__': + utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() + begin, end = time_period(FLAGS.instance_usage_audit_period) + print "Creating usages for %s until %s" % (str(begin), str(end)) + instances = db.instance_get_active_by_window(context.get_admin_context(), + begin, + end) + print "%s instances" % len(instances) + for instance_ref in instances: + usage_info = dict( + tenant_id=instance_ref['project_id'], + user_id=instance_ref['user_id'], + instance_id=instance_ref['id'], + instance_type=instance_ref['instance_type']['name'], + instance_type_id=instance_ref['instance_type_id'], + display_name=instance_ref['display_name'], + created_at=str(instance_ref['created_at']), + launched_at=str(instance_ref['launched_at']) \ + if instance_ref['launched_at'] else '', + image_id=instance_ref['image_id'], + audit_period_begining=str(begin), + audit_period_ending=str(end)) + notifier_api.notify('compute.%s' % FLAGS.host, + 'compute.instance.exists', + notifier_api.INFO, + usage_info) + + diff --git a/nova/notifier/test_notifier.py b/nova/notifier/test_notifier.py new file mode 100644 index 000000000..d43f43e48 --- /dev/null +++ b/nova/notifier/test_notifier.py @@ -0,0 +1,28 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from nova import flags +from nova import log as logging + +FLAGS = flags.FLAGS + +NOTIFICATIONS = [] + + +def notify(message): + """Test notifier, stores notifications in memory for unittests.""" + NOTIFICATIONS.append(message) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 55e7ae0c4..30a65a4b1 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -38,6 +38,7 @@ from nova.compute import manager as compute_manager from nova.compute import power_state from nova.db.sqlalchemy import models from nova.image import local +from nova.notifier import test_notifier LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -63,6 +64,7 @@ class ComputeTestCase(test.TestCase): super(ComputeTestCase, self).setUp() self.flags(connection_type='fake', stub_network=True, + notification_driver='nova.notifier.test_notifier', network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() @@ -70,6 +72,7 @@ class ComputeTestCase(test.TestCase): self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake', False) + test_notifier.NOTIFICATIONS = [] def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} @@ -305,6 +308,50 @@ class ComputeTestCase(test.TestCase): self.assert_(console) self.compute.terminate_instance(self.context, instance_id) + def test_run_instance_usage_notification(self): + """Ensure run instance generates apropriate usage notification""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.assertEquals(len(test_notifier.NOTIFICATIONS), 1) + msg = test_notifier.NOTIFICATIONS[0] + self.assertEquals(msg['priority'], 'INFO') + self.assertEquals(msg['event_type'], 'compute.instance.create') + payload = msg['payload'] + self.assertEquals(payload['tenant_id'], self.project.id) + self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['instance_id'], instance_id) + self.assertEquals(payload['instance_type'], 'm1.tiny') + type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] + self.assertEquals(str(payload['instance_type_id']), str(type_id)) + self.assertTrue('display_name' in payload) + self.assertTrue('created_at' in payload) + self.assertTrue('launched_at' in payload) + self.assertEquals(payload['image_id'], '1') + self.compute.terminate_instance(self.context, instance_id) + + def test_terminate_usage_notification(self): + """Ensure terminate_instance generates apropriate usage notification""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + test_notifier.NOTIFICATIONS = [] + self.compute.terminate_instance(self.context, instance_id) + + self.assertEquals(len(test_notifier.NOTIFICATIONS), 1) + msg = test_notifier.NOTIFICATIONS[0] + self.assertEquals(msg['priority'], 'INFO') + self.assertEquals(msg['event_type'], 'compute.instance.delete') + payload = msg['payload'] + self.assertEquals(payload['tenant_id'], self.project.id) + self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['instance_id'], instance_id) + self.assertEquals(payload['instance_type'], 'm1.tiny') + type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] + self.assertEquals(str(payload['instance_type_id']), str(type_id)) + self.assertTrue('display_name' in payload) + self.assertTrue('created_at' in payload) + self.assertTrue('launched_at' in payload) + self.assertEquals(payload['image_id'], '1') + def test_run_instance_existing(self): """Ensure failure when running an instance that already exists""" instance_id = self._create_instance() @@ -334,6 +381,36 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(self.context, instance_id) + def test_resize_instance_notification(self): + """Ensure instance can be migrated/resized""" + instance_id = self._create_instance() + context = self.context.elevated() + + self.compute.run_instance(self.context, instance_id) + test_notifier.NOTIFICATIONS = [] + + db.instance_update(self.context, instance_id, {'host': 'foo'}) + self.compute.prep_resize(context, instance_id, 1) + migration_ref = db.migration_get_by_instance_and_status(context, + instance_id, 'pre-migrating') + + self.assertEquals(len(test_notifier.NOTIFICATIONS), 1) + msg = test_notifier.NOTIFICATIONS[0] + self.assertEquals(msg['priority'], 'INFO') + self.assertEquals(msg['event_type'], 'compute.instance.resize.prep') + payload = msg['payload'] + self.assertEquals(payload['tenant_id'], self.project.id) + self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['instance_id'], instance_id) + self.assertEquals(payload['instance_type'], 'm1.tiny') + type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] + self.assertEquals(str(payload['instance_type_id']), str(type_id)) + self.assertTrue('display_name' in payload) + self.assertTrue('created_at' in payload) + self.assertTrue('launched_at' in payload) + self.assertEquals(payload['image_id'], '1') + self.compute.terminate_instance(context, instance_id) + def test_resize_instance(self): """Ensure instance can be migrated/resized""" instance_id = self._create_instance() From c6b78050468cc231fa1fc52ff7d934642db07c30 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 28 Jun 2011 09:08:35 +0000 Subject: [PATCH 092/107] Fix pep8 nits in audit script --- bin/instance-usage-audit | 53 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit index 1124cf550..cef9b464b 100755 --- a/bin/instance-usage-audit +++ b/bin/instance-usage-audit @@ -17,22 +17,22 @@ # under the License. """Cron script to generate usage notifications for instances neither created - nor destroyed in a given time period. - + nor destroyed in a given time period. + Together with the notifications generated by compute on instance - create/delete/resize, over that ime period, this allows an external - system consuming usage notification feeds to calculate instance usage + create/delete/resize, over that ime period, this allows an external + system consuming usage notification feeds to calculate instance usage for each tenant. Time periods are specified like so: - [mdy] + [mdy] 1m = previous month. If the script is run April 1, it will generate usages for March 1 thry March 31. 3m = 3 previous months. - 90d = previous 90 days. - 1y = previous year. If run on Jan 1, it generates usages for - Jan 1 thru Dec 31 of the previous year. + 90d = previous 90 days. + 1y = previous year. If run on Jan 1, it generates usages for + Jan 1 thru Dec 31 of the previous year. """ import datetime @@ -65,6 +65,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('instance_usage_audit_period', '1m', 'time period to generate instance usages for.') + def time_period(period): today = datetime.date.today() unit = period[-1] @@ -72,27 +73,27 @@ def time_period(period): raise ValueError('Time period must be m, d, or y') n = int(period[:-1]) if unit == 'm': - year = today.year - (n//12) - n = n % 12 - if n >= today.month: - year -= 1 - month = 12 + (today.month - n) - else: - month = today.month - n - begin = datetime.datetime(day=1, month=month, year=year) - end = datetime.datetime(day=1, month=today.month, year=today.year) + year = today.year - (n // 12) + n = n % 12 + if n >= today.month: + year -= 1 + month = 12 + (today.month - n) + else: + month = today.month - n + begin = datetime.datetime(day=1, month=month, year=year) + end = datetime.datetime(day=1, month=today.month, year=today.year) elif unit == 'y': - begin = datetime.datetime(day=1, month=1, year=today.year - n) - end = datetime.datetime(day=1, month=1, year=today.year) - + begin = datetime.datetime(day=1, month=1, year=today.year - n) + end = datetime.datetime(day=1, month=1, year=today.year) + elif unit == 'd': - b = today - datetime.timedelta(days=n) - begin = datetime.datetime(day=b.day, month=b.month, year=b.year) - end = datetime.datetime(day=today.day, - month=today.month, + b = today - datetime.timedelta(days=n) + begin = datetime.datetime(day=b.day, month=b.month, year=b.year) + end = datetime.datetime(day=today.day, + month=today.month, year=today.year) - + return (begin, end) if __name__ == '__main__': @@ -123,5 +124,3 @@ if __name__ == '__main__': 'compute.instance.exists', notifier_api.INFO, usage_info) - - From 0a4d17c5cca8fee9ba8c83aa7d75bb033d4b512f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 28 Jun 2011 10:43:25 -0400 Subject: [PATCH 093/107] pep8 fix --- nova/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index caa1df22a..918541224 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -790,4 +790,3 @@ class Bootstrapper(object): for key in FLAGS: value = FLAGS.get(key, None) logging.audit(_("%(key)s : %(value)s" % locals())) - From 5ca2d4f6bdad34e07cedb699645a81629bebba03 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 28 Jun 2011 18:52:22 +0400 Subject: [PATCH 094/107] Prevent test case from ruining other tests. Make it work in earlier python versions. --- nova/tests/test_auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 4aebbe940..71e0d17c9 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -373,9 +373,10 @@ class AuthManagerLdapTestCase(_AuthManagerBaseTestCase): def test_reconnect_on_server_failure(self): self.manager.get_users() fakeldap.server_fail = True - with self.assertRaises(fakeldap.SERVER_DOWN): - self.manager.get_users() - fakeldap.server_fail = False + try: + self.assertRaises(fakeldap.SERVER_DOWN, self.manager.get_users) + finally: + fakeldap.server_fail = False self.manager.get_users() From 5506dc3c99812433982856912a86c9462fc9750b Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 28 Jun 2011 08:08:13 -0700 Subject: [PATCH 095/107] log formatting typo pep8 fixes --- nova/scheduler/zone_aware_scheduler.py | 3 ++- nova/tests/scheduler/test_zone_aware_scheduler.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 638072ee7..2efad7bf2 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -181,7 +181,8 @@ class ZoneAwareScheduler(driver.Scheduler): return None num_instances = request_spec.get('num_instances', 1) - LOG.debug(_("Attemping to build %d instance(s)") % locals()) + LOG.debug(_("Attempting to build %(num_instances)d instance(s)") % + locals()) # Create build plan and provision ... build_plan = self.select(context, request_spec) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index b2599f1b8..1e23e3ee6 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -63,13 +63,13 @@ class FakeZoneManager(zone_manager.ZoneManager): def __init__(self): self.service_states = { 'host1': { - 'compute': {'host_memory_free': 1000*1024*1024}, + 'compute': {'host_memory_free': 1073741824}, }, 'host2': { - 'compute': {'host_memory_free': 2000*1024*1024}, + 'compute': {'host_memory_free': 2147483648}, }, 'host3': { - 'compute': {'host_memory_free': 3000*1024*1024}, + 'compute': {'host_memory_free': 3221225472}, }, } @@ -158,7 +158,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase): fake_context = {} build_plan = sched.select(fake_context, {'instance_type': {'memory_mb': 512}, - 'num_instances': 4 }) + 'num_instances': 4}) # 4 from local zones, 12 from remotes self.assertEqual(16, len(build_plan)) From e4e5d8cb33152dc4b0766a28704e13dfec27b1b9 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 28 Jun 2011 08:12:08 -0700 Subject: [PATCH 096/107] update a test docstring to make it clear we're testing multiple instance builds --- nova/tests/scheduler/test_zone_aware_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 1e23e3ee6..32f5150a5 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -146,8 +146,8 @@ class ZoneAwareSchedulerTestCase(test.TestCase): def test_zone_aware_scheduler(self): """ - Create a nested set of FakeZones, ensure that a select call returns the - appropriate build plan. + Create a nested set of FakeZones, try to build multiple instances + and ensure that a select call returns the appropriate build plan. """ sched = FakeZoneAwareScheduler() self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method) From d25eb8be0d6c88e4bb61cb934bb6a7cb8360fc79 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 28 Jun 2011 15:21:08 +0000 Subject: [PATCH 097/107] Fix issues due to renming of imange_id attrib. --- bin/instance-usage-audit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit index cef9b464b..1fa2c2186 100755 --- a/bin/instance-usage-audit +++ b/bin/instance-usage-audit @@ -117,7 +117,7 @@ if __name__ == '__main__': created_at=str(instance_ref['created_at']), launched_at=str(instance_ref['launched_at']) \ if instance_ref['launched_at'] else '', - image_id=instance_ref['image_id'], + image_ref=instance_ref['image_ref'], audit_period_begining=str(begin), audit_period_ending=str(end)) notifier_api.notify('compute.%s' % FLAGS.host, From 10fcb3ea575ec569f1fd192d2f895b5b34bed3c1 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 28 Jun 2011 08:53:13 -0700 Subject: [PATCH 098/107] change variable names to remove future conflict with sandy's zone-offsets branch --- nova/scheduler/zone_aware_scheduler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 2efad7bf2..2e6662a79 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -192,9 +192,10 @@ class ZoneAwareScheduler(driver.Scheduler): for num in xrange(num_instances): if not build_plan: break - item = build_plan.pop(0) - self._provision_resource(context, item, instance_id, - request_spec, kwargs) + + build_plan_item = build_plan.pop(0) + self._provision_resource(context, build_plan_item, instance_id, + request_spec, kwargs) # Returning None short-circuits the routing to Compute (since # we've already done it here) From f8434ec194d6762266e3e1033cbbc2626a286794 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 28 Jun 2011 16:08:27 +0000 Subject: [PATCH 099/107] Fix merge issue in compute unittest. --- nova/tests/test_compute.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e94a62679..4d42b1fdf 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -348,7 +348,7 @@ class ComputeTestCase(test.TestCase): self.assertTrue('display_name' in payload) self.assertTrue('created_at' in payload) self.assertTrue('launched_at' in payload) - self.assertEquals(payload['image_id'], '1') + self.assertEquals(payload['image_ref'], '1') self.compute.terminate_instance(self.context, instance_id) def test_terminate_usage_notification(self): @@ -372,7 +372,7 @@ class ComputeTestCase(test.TestCase): self.assertTrue('display_name' in payload) self.assertTrue('created_at' in payload) self.assertTrue('launched_at' in payload) - self.assertEquals(payload['image_id'], '1') + self.assertEquals(payload['image_ref'], '1') def test_run_instance_existing(self): """Ensure failure when running an instance that already exists""" @@ -452,7 +452,7 @@ class ComputeTestCase(test.TestCase): self.assertTrue('display_name' in payload) self.assertTrue('created_at' in payload) self.assertTrue('launched_at' in payload) - self.assertEquals(payload['image_id'], '1') + self.assertEquals(payload['image_ref'], '1') self.compute.terminate_instance(context, instance_id) def test_resize_instance(self): From ea4193eb06cca2f3f39349809d417703f720f730 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 28 Jun 2011 20:37:05 +0000 Subject: [PATCH 100/107] Refactored usage generation --- bin/instance-usage-audit | 16 +++------------- nova/utils.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit index 1fa2c2186..a06c6b1b3 100755 --- a/bin/instance-usage-audit +++ b/bin/instance-usage-audit @@ -107,19 +107,9 @@ if __name__ == '__main__': end) print "%s instances" % len(instances) for instance_ref in instances: - usage_info = dict( - tenant_id=instance_ref['project_id'], - user_id=instance_ref['user_id'], - instance_id=instance_ref['id'], - instance_type=instance_ref['instance_type']['name'], - instance_type_id=instance_ref['instance_type_id'], - display_name=instance_ref['display_name'], - created_at=str(instance_ref['created_at']), - launched_at=str(instance_ref['launched_at']) \ - if instance_ref['launched_at'] else '', - image_ref=instance_ref['image_ref'], - audit_period_begining=str(begin), - audit_period_ending=str(end)) + usage_info = utils.usage_from_instance(instance_ref, + audit_period_begining=str(begin), + audit_period_ending=str(end)) notifier_api.notify('compute.%s' % FLAGS.host, 'compute.instance.exists', notifier_api.INFO, diff --git a/nova/utils.py b/nova/utils.py index 6d8324e5b..aee2715ba 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -279,6 +279,22 @@ EASIER_PASSWORD_SYMBOLS = ('23456789' # Removed: 0, 1 'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O +def usage_from_instance(instance_ref, **kw): + usage_info = dict( + tenant_id=instance_ref['project_id'], + user_id=instance_ref['user_id'], + instance_id=instance_ref['id'], + instance_type=instance_ref['instance_type']['name'], + instance_type_id=instance_ref['instance_type_id'], + display_name=instance_ref['display_name'], + created_at=str(instance_ref['created_at']), + launched_at=str(instance_ref['launched_at']) \ + if instance_ref['launched_at'] else '', + image_ref=instance_ref['image_ref']) + usage_info.update(kw) + return usage_info + + def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS): """Generate a random password from the supplied symbols. From f3a879d92935b7a5d2640c572038f3137557cd01 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 28 Jun 2011 17:26:08 -0500 Subject: [PATCH 101/107] Use milestone cut. --- bin/nova-api | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index ea99a1b48..fff67251f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -23,8 +23,14 @@ Starts both the EC2 and OpenStack APIs in separate processes. """ +import os import sys +possible_topdir = os.path.normpath(os.path.join(os.path.abspath( + sys.argv[0]), os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, "nova", "__init__.py")): + sys.path.insert(0, possible_topdir) + import nova.service import nova.utils From bb278139a8e8cfb39abd8f3c47e2f4864f07a131 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 29 Jun 2011 16:45:46 +0200 Subject: [PATCH 102/107] Fix nova-manage vm list --- bin/nova-manage | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 02f20347d..d5390b636 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -617,7 +617,7 @@ class VmCommands(object): :param host: show all instance on specified host. :param instance: show specificed instance. """ - print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \ + print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \ " %-10s %-10s %-10s %-5s" % ( _('instance'), _('node'), @@ -639,14 +639,14 @@ class VmCommands(object): context.get_admin_context(), host) for instance in instances: - print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \ + print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \ " %-10s %-10s %-10s %-5d" % ( instance['hostname'], instance['host'], - instance['instance_type'], + instance['instance_type'].name, instance['state_description'], instance['launched_at'], - instance['image_id'], + instance['image_ref'], instance['kernel_id'], instance['ramdisk_id'], instance['project_id'], From e45158490c4c2515bbeab962201fdf6db721936b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 29 Jun 2011 07:47:51 -0700 Subject: [PATCH 103/107] change the default to recreate the db but allow -n for faster tests --- run_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 2ea221ae3..ddeb1dc4a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,7 +6,8 @@ function usage { echo "" echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" - echo " -r, --recreate-db Recreate the test database." + echo " -r, --recreate-db Recreate the test database (deprecated, as this is now the default)." + echo " -n, --no-recreate-db Don't recreate the test database." echo " -x, --stop Stop running tests after the first error or failure." echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" @@ -25,6 +26,7 @@ function process_option { -V|--virtual-env) let always_venv=1; let never_venv=0;; -N|--no-virtual-env) let always_venv=0; let never_venv=1;; -r|--recreate-db) let recreate_db=1;; + -n|--no-recreate-db) let recreate_db=0;; -f|--force) let force=1;; -p|--pep8) let just_pep8=1;; -*) noseopts="$noseopts $1";; @@ -41,7 +43,7 @@ noseargs= noseopts= wrapper="" just_pep8=0 -recreate_db=0 +recreate_db=1 for arg in "$@"; do process_option $arg From 3fb677752987f837c5a4ee5f17fa774fdeef0f74 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 29 Jun 2011 16:52:55 +0200 Subject: [PATCH 104/107] Fix 'undefined name 'e'' pylint error --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index d5390b636..51e0c32c9 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -878,7 +878,7 @@ class InstanceTypeCommands(object): try: instance_types.create(name, memory, vcpus, local_gb, flavorid, swap, rxtx_quota, rxtx_cap) - except exception.InvalidInput: + except exception.InvalidInput, e: print "Must supply valid parameters to create instance_type" print e sys.exit(1) From 7e2d06ecedb60d1d5052b22882a35c9cc2de409d Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 29 Jun 2011 09:49:19 -0700 Subject: [PATCH 105/107] Fanout queues use unique queue names, so the consumer should have exclusive access. This means that they also get auto deleted when we're done with them, so they're not left around on a service restart. Fixes lp:803165 --- nova/rpc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 2e78a31e7..9f0b507fd 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -275,6 +275,11 @@ class FanoutAdapterConsumer(AdapterConsumer): unique = uuid.uuid4().hex self.queue = '%s_fanout_%s' % (topic, unique) self.durable = False + # Fanout creates unique queue names, so we should auto-remove + # them when done, so they're not left around on restart. + # Also, we're the only one that should be consuming. exclusive + # implies auto_delete, so we'll just set that.. + self.exclusive = True LOG.info(_('Created "%(exchange)s" fanout exchange ' 'with "%(key)s" routing key'), dict(exchange=self.exchange, key=self.routing_key)) From e19e0c1fffa1a7ca889768df4a8fbccfda7e7090 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 29 Jun 2011 12:23:26 -0700 Subject: [PATCH 106/107] Fixed indentation issues Fixed min/max_count checking issues Fixed a wrongly log message when zone aware scheduler finds no suitable hosts --- nova/scheduler/zone_aware_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 2e6662a79..9e4fc47d1 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -245,7 +245,7 @@ class ZoneAwareScheduler(driver.Scheduler): # may have been consumed from a previous build.. host_list = self.filter_hosts(topic, request_spec, host_list) if not host_list: - LOG.warn(_("Ran out of available hosts after weighing " + LOG.warn(_("Filter returned no hosts after processing " "%(i)d of %(num_instances)d instances") % locals()) break From bb1244904824960e07807ee9a85169824d1db9f4 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 29 Jun 2011 15:22:56 -0700 Subject: [PATCH 107/107] fix issue of recurse_zones not being converted to bool properly add bool_from_str util call add test for bool_from_str slight rework of min/max_count check --- nova/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nova/utils.py b/nova/utils.py index 510cdd9f6..be26899ca 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -772,6 +772,17 @@ def is_uuid_like(val): return (len(val) == 36) and (val.count('-') == 4) +def bool_from_str(val): + """Convert a string representation of a bool into a bool value""" + + if not val: + return False + try: + return True if int(val) else False + except ValueError: + return val.lower() == 'true' + + class Bootstrapper(object): """Provides environment bootstrapping capabilities for entry points."""