Add ipset element and hashsize tunables

Recently, these messages have been noticed in both tempest
logs, as well as reported by downstream users syslog:

  Set IPv4915d358d-2c5b-43b5-9862 is full, maxelem 65536 reached

So the default of 64K is not sufficient enough.

This change adds two config options to control both the number
of elements as well as the hashsize, since they should be
tuned together for best performance.  Slightly different
formats were required for 'ipset create' and 'ipset restore'.

The default values for these are now set to 131072 (maxelem) and
2048 (hashsize), which is an increase over their typical default values
of 65536/1024 (respectively), in order to fix the errors seen in
the tempest tests.


Change-Id: Ic0b5b38a840e737dc6be938230f4052974c8620f
Closes-bug: #1439817
Brian Haley 2015-04-02 21:11:06 -04:00
parent a38d294f1e
commit b5b919a7a3
4 changed files with 75 additions and 6 deletions

View File

@ -664,6 +664,15 @@ lock_path = $state_path/lock
# each rule's purpose. (System must support the iptables comments module.)
# comment_iptables_rules = True
# Maximum number of elements which can be stored in an IPset.
# If None is specified, the system default will be used.
# ipset_maxelem = 131072
# Initial hash size for an IPset. Must be a power of 2,
# else the kernel will round it up automatically.
# If None is specified, the system default will be used.
# ipset_hashsize = 2048
# Use the root helper when listing the namespaces on a system. This may not
# be required depending on the security configuration. If the root helper is
# not required, set this to False for a performance improvement.

View File

@ -63,6 +63,17 @@ IPTABLES_OPTS = [
help=_("Add comments to iptables rules.")),
cfg.IntOpt('ipset_maxelem', default=131072,
help=_("Maximum number of elements which can be stored in "
"an IPset. If None is specified, the system default "
"will be used.")),
cfg.IntOpt('ipset_hashsize', default=2048,
help=_("Initial hash size for an IPset. Must be a power of 2, "
"else the kernel will round it up automatically. If "
"None is specified, the system default will be used.")),
cfg.StrOpt('check_child_processes_action', default='respawn',
choices=['respawn', 'exit'],
@ -122,6 +133,10 @@ def register_iptables_opts(conf):
conf.register_opts(IPTABLES_OPTS, 'AGENT')
def register_ipset_opts(conf):
conf.register_opts(IPSET_OPTS, 'AGENT')
def register_process_monitor_opts(conf):
conf.register_opts(PROCESS_MONITOR_OPTS, 'AGENT')

View File

@ -11,6 +11,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from neutron.agent.common import config
from neutron.agent.linux import utils as linux_utils
from neutron.common import utils
@ -29,6 +32,7 @@ class IpsetManager(object):
def __init__(self, execute=None, namespace=None):
self.execute = execute or linux_utils.execute
self.namespace = namespace
self.ipset_sets = {}
@ -39,6 +43,15 @@ class IpsetManager(object):
name = ethertype + id
def get_hashargs():
args = []
if cfg.CONF.AGENT.ipset_hashsize:
args.extend(['hashsize', str(cfg.CONF.AGENT.ipset_hashsize)])
if cfg.CONF.AGENT.ipset_maxelem:
args.extend(['maxelem', str(cfg.CONF.AGENT.ipset_maxelem)])
return args
def set_exists(self, id, ethertype):
"""Returns true if the id+ethertype pair is known to the manager."""
set_name = self.get_name(id, ethertype)
@ -85,8 +98,10 @@ class IpsetManager(object):
def _refresh_set(self, set_name, member_ips, ethertype):
new_set_name = set_name + SWAP_SUFFIX
set_type = self._get_ipset_set_type(ethertype)
process_input = ["create %s hash:ip family %s" % (new_set_name,
hash_args = ' '.join(self.get_hashargs())
process_input = ["create %s hash:ip family %s %s" % (new_set_name,
for ip in member_ips:
process_input.append("add %s %s" % (new_set_name, ip))
@ -103,6 +118,7 @@ class IpsetManager(object):
def _create_set(self, set_name, ethertype):
cmd = ['ipset', 'create', '-exist', set_name, 'hash:ip', 'family',
self.ipset_sets[set_name] = []

View File

@ -12,7 +12,9 @@
# limitations under the License.
import mock
from oslo_config import cfg
from neutron.agent.common import config as a_cfg
from neutron.agent.linux import ipset_manager
from neutron.tests import base
@ -25,8 +27,13 @@ FAKE_IPS = ['', '', '', '',
class BaseIpsetManagerTest(base.BaseTestCase):
def setUp(self):
def setUp(self, maxelem=None, hashsize=None):
super(BaseIpsetManagerTest, self).setUp()
cfg.CONF.register_opts(a_cfg.IPSET_OPTS, 'AGENT')
cfg.CONF.set_override('ipset_maxelem', maxelem, 'AGENT')
cfg.CONF.set_override('ipset_hashsize', hashsize, 'AGENT')
self.maxelem = maxelem
self.hashsize = hashsize
self.ipset = ipset_manager.IpsetManager()
self.execute = mock.patch.object(self.ipset, "execute").start()
self.expected_calls = []
@ -36,7 +43,13 @@ class BaseIpsetManagerTest(base.BaseTestCase):
self.execute.assert_has_calls(self.expected_calls, any_order=False)
def expect_set(self, addresses):
temp_input = ['create IPv4fake_sgid-new hash:ip family inet']
hash_args = []
if self.hashsize:
hash_args.extend(['hashsize', str(self.hashsize)])
if self.maxelem:
hash_args.extend(['maxelem', str(self.maxelem)])
temp_input = ['create IPv4fake_sgid-new hash:ip family inet %s' %
' '.join(hash_args)]
temp_input.extend('add IPv4fake_sgid-new %s' % ip for ip in addresses)
input = '\n'.join(temp_input)
@ -63,9 +76,14 @@ class BaseIpsetManagerTest(base.BaseTestCase):
run_as_root=True) for ip in addresses)
def expect_create(self):
ipset_call = ['ipset', 'create', '-exist', TEST_SET_NAME,
'hash:ip', 'family', 'inet']
if self.hashsize:
ipset_call.extend(['hashsize', str(self.hashsize)])
if self.maxelem:
ipset_call.extend(['maxelem', str(self.maxelem)])
self.expected_calls.append(['ipset', 'create', '-exist', TEST_SET_NAME,
'hash:ip', 'family', 'inet'],,
@ -85,6 +103,10 @@ class BaseIpsetManagerTest(base.BaseTestCase):
class IpsetManagerTestCase(BaseIpsetManagerTest):
"""Run all tests, but with maxelem/hashsize values not configured
def setUp(self):
super(IpsetManagerTestCase, self).setUp()
def test_set_exists(self):
@ -117,3 +139,10 @@ class IpsetManagerTestCase(BaseIpsetManagerTest):
self.ipset.destroy(TEST_SET_ID, ETHERTYPE)
class IpsetManagerTestCaseHashArgs(IpsetManagerTestCase):
"""Run all the above tests, but with maxelem/hashsize values configured
def setUp(self):
super(IpsetManagerTestCase, self).setUp(maxelem=131072, hashsize=2048)