mtu support

charm-helpers-hooks.yaml

@@ -1,4 +1,4 @@
-branch: lp:charm-helpers
+branch: lp:~cts-engineering/charm-helpers/neutron-mtu 
 destination: hooks/charmhelpers
 include:
     - core
@@ -8,5 +8,6 @@ include:
     - contrib.network
     - contrib.python.packages
     - contrib.storage.linux
+    - contrib.python
     - payload.execd
     - contrib.charmsupport

config.yaml

@@ -154,4 +154,8 @@ options:
     description: |
       Default multicast port number that will be used to communicate between
       HA Cluster nodes.
-
+  phy-nic-mtu:
+    type: int
+    default: 1500
+      To improve network performance of VM, sometimes we should keep VM MTU as 1500
+      and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE)

hooks/charmhelpers/contrib/network/ip.py

@@ -23,7 +23,14 @@ from functools import partial
 from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    log
+    config,
+    log,
+    INFO
+)
+from charmhelpers.core.host import (
+    list_nics,
+    get_nic_mtu,
+    set_nic_mtu
 )
 
 try:
@@ -365,3 +372,26 @@ def is_bridge_member(nic):
             return True
 
     return False
+
+
+def configure_phy_nic_mtu(mng_ip=None):
+    """Configure mtu for physical nic."""
+    phy_nic_mtu = config('phy-nic-mtu')
+    if phy_nic_mtu >= 1500:
+        phy_nic = None
+        if mng_ip is None:
+            mng_ip = unit_get('private-address')
+        for nic in list_nics(['eth', 'bond', 'br']):
+            if mng_ip in get_ipv4_addr(nic, fatal=False):
+                phy_nic = nic
+                # need to find the associated phy nic for bridge
+                if nic.startswith('br'):
+                    for brnic in get_bridge_nics(nic):
+                        if brnic.startswith('eth') or brnic.startswith('bond'):
+                            phy_nic = brnic
+                            break
+                break
+        if phy_nic is not None and phy_nic_mtu != get_nic_mtu(phy_nic):
+            set_nic_mtu(phy_nic, str(phy_nic_mtu), persistence=True)
+            log('set mtu={} for phy_nic={}'
+                .format(phy_nic_mtu, phy_nic), level=INFO)

hooks/charmhelpers/contrib/network/ufw.py

@@ -53,6 +53,7 @@ def is_enabled():
     :returns: True if ufw is enabled
     """
     output = subprocess.check_output(['ufw', 'status'],
+                                     universal_newlines=True,
                                      env={'LANG': 'en_US',
                                           'PATH': os.environ['PATH']})
 
@@ -82,6 +83,7 @@ def enable():
             raise Exception("Couldn't disable IPv6 support in ufw")
 
     output = subprocess.check_output(['ufw', 'enable'],
+                                     universal_newlines=True,
                                      env={'LANG': 'en_US',
                                           'PATH': os.environ['PATH']})
 
@@ -107,6 +109,7 @@ def disable():
         return True
 
     output = subprocess.check_output(['ufw', 'disable'],
+                                     universal_newlines=True,
                                      env={'LANG': 'en_US',
                                           'PATH': os.environ['PATH']})
 
@@ -151,7 +154,7 @@ def modify_access(src, dst='any', port=None, proto=None, action='allow'):
         cmd += ['to', dst]
 
     if port is not None:
-        cmd += ['port', port]
+        cmd += ['port', str(port)]
 
     if proto is not None:
         cmd += ['proto', proto]
@@ -208,9 +211,11 @@ def service(name, action):
     :param action: `open` or `close`
     """
     if action == 'open':
-        subprocess.check_output(['ufw', 'allow', name])
+        subprocess.check_output(['ufw', 'allow', str(name)],
+                                universal_newlines=True)
     elif action == 'close':
-        subprocess.check_output(['ufw', 'delete', 'allow', name])
+        subprocess.check_output(['ufw', 'delete', 'allow', str(name)],
+                                universal_newlines=True)
     else:
         raise Exception(("'{}' not supported, use 'allow' "
                          "or 'delete'").format(action))

hooks/charmhelpers/contrib/python/debug.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+import atexit
+import sys
+
+from charmhelpers.contrib.python.rpdb import Rpdb
+from charmhelpers.core.hookenv import (
+    open_port,
+    close_port,
+    ERROR,
+    log
+)
+
+DEFAULT_ADDR = "0.0.0.0"
+DEFAULT_PORT = 4444
+
+
+def _error(message):
+    log(message, level=ERROR)
+
+
+def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
+    """
+    Set a trace point using the remote debugger
+    """
+    atexit.register(close_port, port)
+    try:
+        log("Starting a remote python debugger session on %s:%s" % (addr,
+                                                                    port))
+        open_port(port)
+        debugger = Rpdb(addr=addr, port=port)
+        debugger.set_trace(sys._getframe().f_back)
+    except:
+        _error("Cannot start a remote debug session on %s:%s" % (addr,
+                                                                 port))

hooks/charmhelpers/contrib/python/rpdb.py

@@ -0,0 +1,58 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Remote Python Debugger (pdb wrapper)."""
+
+__author__ = "Bertrand Janin <b@janin.com>"
+__version__ = "0.1.3"
+
+import pdb
+import socket
+import sys
+
+
+class Rpdb(pdb.Pdb):
+
+    def __init__(self, addr="127.0.0.1", port=4444):
+        """Initialize the socket and initialize pdb."""
+
+        # Backup stdin and stdout before replacing them by the socket handle
+        self.old_stdout = sys.stdout
+        self.old_stdin = sys.stdin
+
+        # Open a 'reusable' socket to let the webapp reload on the same port
+        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+        self.skt.bind((addr, port))
+        self.skt.listen(1)
+        (clientsocket, address) = self.skt.accept()
+        handle = clientsocket.makefile('rw')
+        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
+        sys.stdout = sys.stdin = handle
+
+    def shutdown(self):
+        """Revert stdin and stdout, close the socket."""
+        sys.stdout = self.old_stdout
+        sys.stdin = self.old_stdin
+        self.skt.close()
+        self.set_continue()
+
+    def do_continue(self, arg):
+        """Stop all operation on ``continue``."""
+        self.shutdown()
+        return 1
+
+    do_EOF = do_quit = do_exit = do_c = do_cont = do_continue

hooks/charmhelpers/contrib/python/version.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+import sys
+
+
+def current_version():
+    """Current system python version"""
+    return sys.version_info
+
+
+def current_version_string():
+    """Current system python version as string major.minor.micro"""
+    return "{0}.{1}.{2}".format(sys.version_info.major,
+                                sys.version_info.minor,
+                                sys.version_info.micro)

hooks/charmhelpers/core/host.py

@@ -361,7 +361,7 @@ def list_nics(nic_type):
         ip_output = (line for line in ip_output if line)
         for line in ip_output:
             if line.split()[1].startswith(int_type):
-                matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
+                matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
                 if matched:
                     interface = matched.groups()[0]
                 else:
@@ -371,10 +371,40 @@ def list_nics(nic_type):
     return interfaces
 
 
-def set_nic_mtu(nic, mtu):
+def set_nic_mtu(nic, mtu, persistence=False):
     '''Set MTU on a network interface'''
     cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
     subprocess.check_call(cmd)
+    # persistence mtu configuration
+    if not persistence:
+        return
+    if os.path.exists("/etc/network/interfaces.d/%s.cfg" % nic):
+        nic_cfg_file = "/etc/network/interfaces.d/%s.cfg" % nic
+    else:
+        nic_cfg_file = "/etc/network/interfaces"
+
+    f = open(nic_cfg_file, "r")
+    lines = f.readlines()
+    found = False
+    length = len(lines)
+    for i in range(len(lines)):
+        lines[i] = lines[i].replace('\n', '')
+        if lines[i].startswith("iface %s" % nic):
+            found = True
+            lines.insert(i + 1, "    up ip link set $IFACE mtu %s" % mtu)
+            lines.insert(i + 2, "    down ip link set $IFACE mtu 1500")
+            if length > i + 2 and lines[i + 3].startswith("    up ip link set $IFACE mtu"):
+                del lines[i + 3]
+            if length > i + 2 and lines[i + 3].startswith("    down ip link set $IFACE mtu"):
+                del lines[i + 3]
+            break
+    if not found:
+        lines.insert(length + 1, "")
+        lines.insert(length + 2, "auto %s" % nic)
+        lines.insert(length + 3, "iface %s inet dhcp" % nic)
+        lines.insert(length + 4, "    up ip link set $IFACE mtu %s" % mtu)
+        lines.insert(length + 5, "    down ip link set $IFACE mtu 1500")
+    write_file(path=nic_cfg_file, content="\n".join(lines), perms=0o644)
 
 
 def get_nic_mtu(nic):

hooks/charmhelpers/core/sysctl.py

@@ -26,25 +26,31 @@ from subprocess import check_call
 from charmhelpers.core.hookenv import (
     log,
     DEBUG,
+    ERROR,
 )
 
 
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
-    :type sysctl_dict: dict
+    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
-    sysctl_dict = yaml.load(sysctl_dict)
+    try:
+        sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+    except yaml.YAMLError:
+        log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+            level=ERROR)
+        return
 
     with open(sysctl_file, "w") as fd:
-        for key, value in sysctl_dict.items():
+        for key, value in sysctl_dict_parsed.items():
             fd.write("{}={}\n".format(key, value))
 
-    log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
+    log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
         level=DEBUG)
 
     check_call(["sysctl", "-p", sysctl_file])

hooks/quantum_contexts.py

@@ -111,8 +111,8 @@ def _neutron_api_settings():
     '''
     neutron_settings = {
         'l2_population': False,
+        'network_device_mtu': 1500,
         'overlay_network_type': 'gre',
-
     }
     for rid in relation_ids('neutron-plugin-api'):
         for unit in related_units(rid):
@@ -122,6 +122,8 @@ def _neutron_api_settings():
             neutron_settings = {
                 'l2_population': rdata['l2-population'],
                 'overlay_network_type': rdata['overlay-network-type'],
+                'network_device_mtu': rdata['network-device-mtu']
+                if 'network-device-mtu' in rdata else 1500,
             }
             return neutron_settings
     return neutron_settings
@@ -243,6 +245,7 @@ class QuantumGatewayContext(OSContextGenerator):
             'verbose': config('verbose'),
             'instance_mtu': config('instance-mtu'),
             'l2_population': neutron_api_settings['l2_population'],
+            'network_device_mtu': neutron_api_settings['network_device_mtu'],
             'overlay_network_type':
             neutron_api_settings['overlay_network_type'],
         }

hooks/quantum_hooks.py

@@ -22,6 +22,9 @@ from charmhelpers.core.host import (
     restart_on_change,
     lsb_release,
 )
+from charmhelpers.contrib.network.ip import (
+    configure_phy_nic_mtu
+)
 from charmhelpers.contrib.hahelpers.cluster import(
     get_hacluster_config,
     eligible_leader
@@ -78,6 +81,7 @@ def install():
                     fatal=True)
         apt_install(filter_installed_packages(get_packages()),
                     fatal=True)
+        configure_phy_nic_mtu()
     else:
         log('Please provide a valid plugin config', level=ERROR)
         sys.exit(1)
@@ -110,6 +114,7 @@ def config_changed():
     if valid_plugin():
         CONFIGS.write_all()
         configure_ovs()
+        configure_phy_nic_mtu()
     else:
         log('Please provide a valid plugin config', level=ERROR)
         sys.exit(1)

templates/icehouse/neutron.conf

@@ -11,5 +11,6 @@ core_plugin = {{ core_plugin }}
 control_exchange = neutron
 notification_driver = neutron.openstack.common.notifier.list_notifier
 list_notifier_drivers = neutron.openstack.common.notifier.rabbit_notifier
+network_device_mtu = {{ network_device_mtu }}
 [agent]
 root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf

unit_tests/test_quantum_contexts.py

@@ -46,11 +46,39 @@ def patch_open():
         yield mock_open, mock_file
 
 
-class _TestQuantumContext(CharmTestCase):
+class TestNetworkServiceContext(CharmTestCase):
 
     def setUp(self):
-        super(_TestQuantumContext, self).setUp(quantum_contexts, TO_PATCH)
+        super(TestNetworkServiceContext, self).setUp(quantum_contexts,
+                                                     TO_PATCH)
         self.config.side_effect = self.test_config.get
+        self.context = quantum_contexts.NetworkServiceContext()
+        self.test_relation.set(
+            {'keystone_host': '10.5.0.1',
+             'service_port': '5000',
+             'auth_port': '20000',
+             'service_tenant': 'tenant',
+             'service_username': 'username',
+             'service_password': 'password',
+             'quantum_host': '10.5.0.2',
+             'quantum_port': '9696',
+             'quantum_url': 'http://10.5.0.2:9696/v2',
+             'region': 'aregion'}
+        )
+        self.data_result = {
+            'keystone_host': '10.5.0.1',
+            'service_port': '5000',
+            'auth_port': '20000',
+            'service_tenant': 'tenant',
+            'service_username': 'username',
+            'service_password': 'password',
+            'quantum_host': '10.5.0.2',
+            'quantum_port': '9696',
+            'quantum_url': 'http://10.5.0.2:9696/v2',
+            'region': 'aregion',
+            'service_protocol': 'http',
+            'auth_protocol': 'http',
+        }
 
     def test_not_related(self):
         self.relation_ids.return_value = []
@@ -84,39 +112,6 @@ class _TestQuantumContext(CharmTestCase):
         self.assertEquals(self.context(), self.data_result)
 
 
-class TestNetworkServiceContext(_TestQuantumContext):
-
-    def setUp(self):
-        super(TestNetworkServiceContext, self).setUp()
-        self.context = quantum_contexts.NetworkServiceContext()
-        self.test_relation.set(
-            {'keystone_host': '10.5.0.1',
-             'service_port': '5000',
-             'auth_port': '20000',
-             'service_tenant': 'tenant',
-             'service_username': 'username',
-             'service_password': 'password',
-             'quantum_host': '10.5.0.2',
-             'quantum_port': '9696',
-             'quantum_url': 'http://10.5.0.2:9696/v2',
-             'region': 'aregion'}
-        )
-        self.data_result = {
-            'keystone_host': '10.5.0.1',
-            'service_port': '5000',
-            'auth_port': '20000',
-            'service_tenant': 'tenant',
-            'service_username': 'username',
-            'service_password': 'password',
-            'quantum_host': '10.5.0.2',
-            'quantum_port': '9696',
-            'quantum_url': 'http://10.5.0.2:9696/v2',
-            'region': 'aregion',
-            'service_protocol': 'http',
-            'auth_protocol': 'http',
-        }
-
-
 class TestNeutronPortContext(CharmTestCase):
 
     def setUp(self):
@@ -241,6 +236,7 @@ class TestQuantumGatewayContext(CharmTestCase):
             'debug': False,
             'verbose': True,
             'l2_population': False,
+            'network_device_mtu': 1500,
             'overlay_network_type': 'gre',
         })
 
@@ -367,24 +363,29 @@ class TestMisc(CharmTestCase):
         self.relation_ids.return_value = ['foo']
         self.related_units.return_value = ['bar']
         self.test_relation.set({'l2-population': True,
+                                'network-device-mtu': 1500,
                                 'overlay-network-type': 'gre', })
         self.relation_get.side_effect = self.test_relation.get
         self.assertEquals(quantum_contexts._neutron_api_settings(),
                           {'l2_population': True,
+                           'network_device_mtu': 1500,
                            'overlay_network_type': 'gre'})
 
     def test_neutron_api_settings2(self):
         self.relation_ids.return_value = ['foo']
         self.related_units.return_value = ['bar']
         self.test_relation.set({'l2-population': True,
+                                'network-device-mtu': 1500,
                                 'overlay-network-type': 'gre', })
         self.relation_get.side_effect = self.test_relation.get
         self.assertEquals(quantum_contexts._neutron_api_settings(),
                           {'l2_population': True,
+                           'network_device_mtu': 1500,
                            'overlay_network_type': 'gre'})
 
     def test_neutron_api_settings_no_apiplugin(self):
         self.relation_ids.return_value = []
         self.assertEquals(quantum_contexts._neutron_api_settings(),
                           {'l2_population': False,
+                           'network_device_mtu': 1500,
                            'overlay_network_type': 'gre', })

unit_tests/test_quantum_hooks.py

@@ -47,7 +47,8 @@ TO_PATCH = [
     'get_hacluster_config',
     'remove_legacy_ha_files',
     'cleanup_ovs_netns',
-    'stop_neutron_ha_monitor_daemon'
+    'stop_neutron_ha_monitor_daemon',
+    'configure_phy_nic_mtu'
 ]
 
 
@@ -87,6 +88,7 @@ class TestQuantumHooks(CharmTestCase):
         self.assertTrue(self.get_early_packages.called)
         self.assertTrue(self.get_packages.called)
         self.assertTrue(self.execd_preinstall.called)
+        self.assertTrue(self.configure_phy_nic_mtu.called)
 
     def test_install_hook_precise_nocloudarchive(self):
         self.test_config.set('openstack-origin', 'distro')
@@ -121,6 +123,7 @@ class TestQuantumHooks(CharmTestCase):
         self.assertTrue(_amqp_joined.called)
         self.assertTrue(_amqp_nova_joined.called)
         self.create_sysctl.assert_called()
+        self.assertTrue(self.configure_phy_nic_mtu.called)
 
     def test_config_changed_upgrade(self):
         self.openstack_upgrade_available.return_value = True
@@ -128,6 +131,7 @@ class TestQuantumHooks(CharmTestCase):
         self._call_hook('config-changed')
         self.assertTrue(self.do_openstack_upgrade.called)
         self.assertTrue(self.configure_ovs.called)
+        self.assertTrue(self.configure_phy_nic_mtu.called)
 
     def test_config_changed_n1kv(self):
         self.openstack_upgrade_available.return_value = False