diff --git a/bin/nova-manage b/bin/nova-manage index 9d67ed95bc5f..0d560ec072ad 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -56,11 +56,11 @@ import gettext import glob import json +import netaddr import os import sys import time -import IPy # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -525,14 +525,14 @@ class FloatingIpCommands(object): def create(self, range): """Creates floating ips for zone by range arguments: ip_range""" - for address in IPy.IP(range): + for address in netaddr.IPNetwork(range): db.floating_ip_create(context.get_admin_context(), {'address': str(address)}) def delete(self, ip_range): """Deletes floating ips by range arguments: range""" - for address in IPy.IP(ip_range): + for address in netaddr.IPNetwork(ip_range): db.floating_ip_destroy(context.get_admin_context(), str(address)) diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index 343bc61c4d19..df7876b9d756 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -369,3 +369,23 @@ class AdminController(object): raise exception.ApiError(_('Duplicate rule')) self.compute_api.trigger_provider_fw_rules_refresh(context) return {'status': 'OK', 'message': 'Added %s rules' % rules_added} + + def describe_external_address_blocks(self, context): + blocks = db.provider_fw_rule_get_all(context) + # NOTE(todd): use a set since we have icmp/udp/tcp rules with same cidr + blocks = set([b.cidr for b in blocks]) + blocks = [{'cidr': b} for b in blocks] + return {'externalIpBlockInfo': + list(sorted(blocks, key=lambda k: k['cidr']))} + + def remove_external_address_block(self, context, cidr): + LOG.audit(_('Removing ip block from %s'), cidr, context=context) + cidr = urllib.unquote(cidr).decode() + # raise if invalid + netaddr.IPNetwork(cidr) + rules = db.provider_fw_rule_get_all_by_cidr(context, cidr) + for rule in rules: + db.provider_fw_rule_destroy(context, rule['id']) + if rules: + self.compute_api.trigger_provider_fw_rules_refresh(context) + return {'status': 'OK', 'message': 'Deleted %s rules' % len(rules)} diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 078d35887c55..e7f94830fd5b 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -23,7 +23,7 @@ datastore. """ import base64 -import IPy +import netaddr import os import urllib import tempfile @@ -452,7 +452,7 @@ class CloudController(object): elif cidr_ip: # If this fails, it throws an exception. This is what we want. cidr_ip = urllib.unquote(cidr_ip).decode() - IPy.IP(cidr_ip) + netaddr.IPNetwork(cidr_ip) values['cidr'] = cidr_ip else: values['cidr'] = '0.0.0.0/0' diff --git a/nova/db/api.py b/nova/db/api.py index a3a3d2b6b54f..b372ac6a78a1 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -1094,6 +1094,16 @@ def provider_fw_rule_get_all(context): return IMPL.provider_fw_rule_get_all(context) +def provider_fw_rule_get_all_by_cidr(context, cidr): + """Get all provider-level firewall rules.""" + return IMPL.provider_fw_rule_get_all_by_cidr(context, cidr) + + +def provider_fw_rule_destroy(context, rule_id): + """Delete a provider firewall rule from the database.""" + return IMPL.provider_fw_rule_destroy(context, rule_id) + + ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index c3de412d7385..d13efb874d39 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2421,6 +2421,7 @@ def provider_fw_rule_create(context, rule): return fw_rule_ref +@require_admin_context def provider_fw_rule_get_all(context): session = get_session() return session.query(models.ProviderFirewallRule).\ @@ -2428,6 +2429,26 @@ def provider_fw_rule_get_all(context): all() +@require_admin_context +def provider_fw_rule_get_all_by_cidr(context, cidr): + session = get_session() + return session.query(models.ProviderFirewallRule).\ + filter_by(deleted=can_read_deleted(context)).\ + filter_by(cidr=cidr).\ + all() + + +@require_admin_context +def provider_fw_rule_destroy(context, rule_id): + session = get_session() + with session.begin(): + session.query(models.ProviderFirewallRule).\ + filter_by(id=rule_id).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + ################### diff --git a/nova/image/glance.py b/nova/image/glance.py index 6e058ab2ff06..55d948a3258a 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -59,7 +59,7 @@ class GlanceImageService(service.BaseImageService): """Provides storage and retrieval of disk image objects within Glance.""" GLANCE_ONLY_ATTRS = ['size', 'location', 'disk_format', - 'container_format'] + 'container_format', 'checksum'] # NOTE(sirp): Overriding to use _translate_to_service provided by # BaseImageService diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9a5f2c5e6bfa..283a5aca10ff 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -20,6 +20,7 @@ import calendar import inspect +import netaddr import os from nova import db @@ -27,7 +28,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import utils -from IPy import IP LOG = logging.getLogger("nova.linux_net") @@ -707,7 +707,7 @@ def _dnsmasq_cmd(net): '--listen-address=%s' % net['gateway'], '--except-interface=lo', '--dhcp-range=%s,static,120s' % net['dhcp_start'], - '--dhcp-lease-max=%s' % IP(net['cidr']).len(), + '--dhcp-lease-max=%s' % len(netaddr.IPNetwork(net['cidr'])), '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), '--dhcp-script=%s' % FLAGS.dhcpbridge, '--leasefile-ro'] diff --git a/nova/network/manager.py b/nova/network/manager.py index 100b5ddc026e..b60e70990fa6 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -47,12 +47,11 @@ topologies. All of the network commands are issued to a subclass of import datetime import math +import netaddr import socket import pickle from eventlet import greenpool -import IPy - from nova import context from nova import db from nova import exception @@ -550,8 +549,8 @@ class NetworkManager(manager.SchedulerDependentManager): network_size, cidr_v6, gateway_v6, bridge, bridge_interface, **kwargs): """Create networks based on parameters.""" - fixed_net = IPy.IP(cidr) - fixed_net_v6 = IPy.IP(cidr_v6) + fixed_net = netaddr.IPNetwork(cidr) + fixed_net_v6 = netaddr.IPNetwork(cidr_v6) significant_bits_v6 = 64 network_size_v6 = 1 << 64 for index in range(num_networks): @@ -559,16 +558,16 @@ class NetworkManager(manager.SchedulerDependentManager): start_v6 = index * network_size_v6 significant_bits = 32 - int(math.log(network_size, 2)) cidr = '%s/%s' % (fixed_net[start], significant_bits) - project_net = IPy.IP(cidr) + project_net = netaddr.IPNetwork(cidr) net = {} net['bridge'] = bridge net['bridge_interface'] = bridge_interface net['dns'] = FLAGS.flat_network_dns net['cidr'] = cidr - net['netmask'] = str(project_net.netmask()) - net['gateway'] = str(project_net[1]) - net['broadcast'] = str(project_net.broadcast()) - net['dhcp_start'] = str(project_net[2]) + net['netmask'] = str(project_net.netmask) + net['gateway'] = str(list(project_net)[1]) + net['broadcast'] = str(project_net.broadcast) + net['dhcp_start'] = str(list(project_net)[2]) if num_networks > 1: net['label'] = '%s_%d' % (label, index) else: @@ -578,15 +577,16 @@ class NetworkManager(manager.SchedulerDependentManager): cidr_v6 = '%s/%s' % (fixed_net_v6[start_v6], significant_bits_v6) net['cidr_v6'] = cidr_v6 - project_net_v6 = IPy.IP(cidr_v6) + + project_net_v6 = netaddr.IPNetwork(cidr_v6) if gateway_v6: # use a pre-defined gateway if one is provided - net['gateway_v6'] = str(gateway_v6) + net['gateway_v6'] = str(list(gateway_v6)[1]) else: - net['gateway_v6'] = str(project_net_v6[1]) + net['gateway_v6'] = str(list(project_net_v6)[1]) - net['netmask_v6'] = str(project_net_v6.prefixlen()) + net['netmask_v6'] = str(project_net_v6._prefixlen) if kwargs.get('vpn', False): # this bit here is for vlan-manager @@ -627,7 +627,7 @@ class NetworkManager(manager.SchedulerDependentManager): # to properties of the manager class? bottom_reserved = self._bottom_reserved_ips top_reserved = self._top_reserved_ips - project_net = IPy.IP(network['cidr']) + project_net = netaddr.IPNetwork(network['cidr']) num_ips = len(project_net) for index in range(num_ips): address = str(project_net[index]) @@ -845,8 +845,8 @@ class VlanManager(RPCAllocateFixedIP, FloatingIP, NetworkManager): ' than 4094')) # check that num networks and network size fits in fixed_net - fixed_net = IPy.IP(kwargs['cidr']) - if fixed_net.len() < kwargs['num_networks'] * kwargs['network_size']: + fixed_net = netaddr.IPNetwork(kwargs['cidr']) + if len(fixed_net) < kwargs['num_networks'] * kwargs['network_size']: raise ValueError(_('The network range is not big enough to fit ' '%(num_networks)s. Network size is %(network_size)s') % kwargs) diff --git a/nova/network/xenapi_net.py b/nova/network/xenapi_net.py index 34a598eadd85..e86f4017de25 100644 --- a/nova/network/xenapi_net.py +++ b/nova/network/xenapi_net.py @@ -56,8 +56,10 @@ def ensure_vlan_bridge(vlan_num, bridge, bridge_interface, net_attrs=None): 'other_config': {}} network_ref = session.call_xenapi('network.create', network_rec) # 2 - find PIF for VLAN - expr = "field 'device' = '%s' and \ - field 'VLAN' = '-1'" % bridge_interface + # NOTE(salvatore-orlando): using double quotes inside single quotes + # as xapi filter only support tokens in double quotes + expr = 'field "device" = "%s" and \ + field "VLAN" = "-1"' % bridge_interface pifs = session.call_xenapi('PIF.get_all_records_where', expr) pif_ref = None # Multiple PIF are ok: we are dealing with a pool diff --git a/nova/tests/api/openstack/test_image_metadata.py b/nova/tests/api/openstack/test_image_metadata.py index efed44960aba..730af366561a 100644 --- a/nova/tests/api/openstack/test_image_metadata.py +++ b/nova/tests/api/openstack/test_image_metadata.py @@ -38,6 +38,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image1', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -53,6 +54,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image2', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -68,6 +70,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image3', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -104,7 +107,10 @@ class ImageMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) - self.assertEqual('value1', res_dict['metadata']['key1']) + expected = self.IMAGE_FIXTURES[0]['properties'] + self.assertEqual(len(expected), len(res_dict['metadata'])) + for (key, value) in res_dict['metadata'].items(): + self.assertEqual(value, res_dict['metadata'][key]) def test_index_xml(self): serializer = openstack.image_metadata.ImageMetadataXMLSerializer() diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index eb4466781fee..877cf4ea1bc4 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -81,5 +81,27 @@ class AdminApiTestCase(test.TestCase): def test_block_external_ips(self): """Make sure provider firewall rules are created.""" result = self.api.block_external_addresses(self.context, '1.1.1.1/32') + self.api.remove_external_address_block(self.context, '1.1.1.1/32') self.assertEqual('OK', result['status']) self.assertEqual('Added 3 rules', result['message']) + + def test_list_blocked_ips(self): + """Make sure we can see the external blocks that exist.""" + self.api.block_external_addresses(self.context, '1.1.1.2/32') + result = self.api.describe_external_address_blocks(self.context) + num = len(db.provider_fw_rule_get_all(self.context)) + self.api.remove_external_address_block(self.context, '1.1.1.2/32') + # we only list IP, not tcp/udp/icmp rules + self.assertEqual(num / 3, len(result['externalIpBlockInfo'])) + + def test_remove_ip_block(self): + """Remove ip blocks.""" + result = self.api.block_external_addresses(self.context, '1.1.1.3/32') + self.assertEqual('OK', result['status']) + num0 = len(db.provider_fw_rule_get_all(self.context)) + result = self.api.remove_external_address_block(self.context, + '1.1.1.3/32') + self.assertEqual('OK', result['status']) + self.assertEqual('Deleted 3 rules', result['message']) + num1 = len(db.provider_fw_rule_get_all(self.context)) + self.assert_(num1 < num0) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 176c536b9c79..5a081ddf7c3d 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -1145,6 +1145,13 @@ class IptablesFirewallTestCase(test.TestCase): provjump_rules.append(rule) self.assertEqual(1, len(provjump_rules)) + # remove a rule from the db, cast to compute to refresh rule + db.provider_fw_rule_destroy(admin_ctxt, provider_fw1['id']) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(1, len(rules)) + class NWFilterTestCase(test.TestCase): def setUp(self): diff --git a/nova/utils.py b/nova/utils.py index b49849f9ad37..4a66dd9933b0 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -518,6 +518,16 @@ def loads(s): return json.loads(s) +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append(("nova.utils", "dumps", TypeError, + "loads", ValueError)) + anyjson.force_implementation("nova.utils") + + _semaphores = {} diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py index 394d60f71819..785a83584177 100644 --- a/nova/virt/libvirt/connection.py +++ b/nova/virt/libvirt/connection.py @@ -38,6 +38,7 @@ Supports KVM, LXC, QEMU, UML, and XEN. import hashlib import multiprocessing +import netaddr import os import random import re @@ -53,8 +54,6 @@ from xml.etree import ElementTree from eventlet import greenthread from eventlet import tpool -import IPy - from nova import context from nova import db from nova import exception diff --git a/nova/virt/libvirt/netutils.py b/nova/virt/libvirt/netutils.py index 773f0a09c795..e5aaf7ceca7e 100644 --- a/nova/virt/libvirt/netutils.py +++ b/nova/virt/libvirt/netutils.py @@ -21,7 +21,7 @@ """Network-releated utilities for supporting libvirt connection code.""" -import IPy +import netaddr from nova import context from nova import db @@ -34,18 +34,18 @@ FLAGS = flags.FLAGS def get_net_and_mask(cidr): - net = IPy.IP(cidr) - return str(net.net()), str(net.netmask()) + net = netaddr.IPNetwork(cidr) + return str(net.ip), str(net.netmask) def get_net_and_prefixlen(cidr): - net = IPy.IP(cidr) - return str(net.net()), str(net.prefixlen()) + net = netaddr.IPNetwork(cidr) + return str(net.ip), str(net._prefixlen) def get_ip_version(cidr): - net = IPy.IP(cidr) - return int(net.version()) + net = netaddr.IPNetwork(cidr) + return int(net.version) def get_network_info(instance): diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch index feaf1312dd92..d42a11effaed 100644 --- a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch +++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch @@ -8,7 +8,7 @@ fi ;; -@@ -224,9 +225,11 @@ +@@ -224,6 +225,7 @@ remove) if [ "${TYPE}" = "vif" ] ;then @@ -16,7 +16,3 @@ xenstore-rm "${HOTPLUG}/hotplug" fi logger -t scripts-vif "${dev} has been removed" - remove_from_bridge - ;; - esac -+ diff --git a/tools/pip-requires b/tools/pip-requires index 0da473109e82..5d31a814da2e 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,7 +1,6 @@ SQLAlchemy==0.6.3 pep8==0.5.0 pylint==0.19 -IPy==0.70 Cheetah==2.4.4 M2Crypto==0.20.2 amqplib==0.6.1