Set IPset hash type to 'net' instead of 'ip'

The previous hash type was 'ip' and this caused a major
issue with the allowed address pairs extension since it
results in CIDRs being passed to ipset. When the hash type
is 'ip', a CIDR is completely enumerated into all of its
addresses so 10.100.0.0/16 results in ~65k entries. This
meant a single allowed_address_pairs entry could easily
exhaust an entire set.

This patch changes the hash type to 'net', which is designed
to handle a CIDRs as a single entry.

This patch also changes the names of the ipsets because
creating an ipset with different parameters will cause an
error and our ipset manager code isn't robust enough to handle
that at this time. There is another ongoing patch to fix
that but it won't be ready in time.[1]

The related bug was closed by increasing the set limit, which
did alleviate the problem. However, this change would also
address the issue because the gate tests run an allowed address
pairs extension test with the CIDR mentioned above.

1. I59e2e1c090cb95ee1bd14dbb53b6ff2c5e2713fd

Related-Bug: #1439817
Closes-Bug: #1444397
Change-Id: I8177699b157cd3eac46e2f481f47b5d966c49b07

neutron/agent/linux/ipset_manager.py

@@ -38,7 +38,7 @@ class IpsetManager(object):
         """Returns the given ipset name for an id+ethertype pair.
         This reference can be used from iptables.
         """
-        name = ethertype + id
+        name = 'NET' + ethertype + id
         return name[:IPSET_NAME_MAX_LENGTH]
 
     def set_exists(self, id, ethertype):
@@ -87,7 +87,7 @@ class IpsetManager(object):
     def _refresh_set(self, set_name, member_ips, ethertype):
         new_set_name = set_name + SWAP_SUFFIX
         set_type = self._get_ipset_set_type(ethertype)
-        process_input = ["create %s hash:ip family %s" % (new_set_name,
+        process_input = ["create %s hash:net family %s" % (new_set_name,
                                                           set_type)]
         for ip in member_ips:
             process_input.append("add %s %s" % (new_set_name, ip))
@@ -103,7 +103,7 @@ class IpsetManager(object):
         self.ipset_sets[set_name].remove(member_ip)
 
     def _create_set(self, set_name, ethertype):
-        cmd = ['ipset', 'create', '-exist', set_name, 'hash:ip', 'family',
+        cmd = ['ipset', 'create', '-exist', set_name, 'hash:net', 'family',
                self._get_ipset_set_type(ethertype)]
         self._apply(cmd)
         self.ipset_sets[set_name] = []

neutron/tests/unit/agent/linux/test_ipset_manager.py

@@ -36,8 +36,9 @@ class BaseIpsetManagerTest(base.BaseTestCase):
         self.execute.assert_has_calls(self.expected_calls, any_order=False)
 
     def expect_set(self, addresses):
-        temp_input = ['create IPv4fake_sgid-new hash:ip family inet']
-        temp_input.extend('add IPv4fake_sgid-new %s' % ip for ip in addresses)
+        temp_input = ['create NETIPv4fake_sgid-new hash:net family inet']
+        temp_input.extend('add NETIPv4fake_sgid-new %s' % ip
+                          for ip in addresses)
         input = '\n'.join(temp_input)
         self.expected_calls.extend([
             mock.call(['ipset', 'restore', '-exist'],
@@ -65,7 +66,7 @@ class BaseIpsetManagerTest(base.BaseTestCase):
     def expect_create(self):
         self.expected_calls.append(
             mock.call(['ipset', 'create', '-exist', TEST_SET_NAME,
-                       'hash:ip', 'family', 'inet'],
+                       'hash:net', 'family', 'inet'],
                       process_input=None,
                       run_as_root=True))
 

neutron/tests/unit/agent/test_securitygroups_rpc.py

@@ -1775,7 +1775,7 @@ IPSET_FILTER_1 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
 -j RETURN
 [0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_port1 -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
@@ -1934,7 +1934,7 @@ IPSET_FILTER_2 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
@@ -1962,7 +1962,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
@@ -2017,7 +2017,7 @@ IPSET_FILTER_2_3 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
@@ -2046,7 +2046,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback