Fixes Octavia handling of subnets without DHCP

Currently Octavia assumes that DHCP service is available on
the VIP and member subnets.  This is not the case at all operators.
This patch makes Octavia use the IP information provided when
the ports are created, if available.  If the IP information is
not available on the ports it will fall back to relying on DHCP.

Change-Id: I08a93d4318bbce48128019376320782d1a334369
Closes-Bug: #1607900

octavia/amphorae/backends/agent/api_server/plug.py

@@ -1,4 +1,5 @@
 # Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2016 Rackspace
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -20,9 +21,12 @@ import stat
 import subprocess
 
 import flask
+import ipaddress
 import jinja2
 import netifaces
+from oslo_config import cfg
 import pyroute2
+import six
 from werkzeug import exceptions
 
 from octavia.amphorae.backends.agent.api_server import util
@@ -30,6 +34,9 @@ from octavia.common import constants as consts
 from octavia.i18n import _LE, _LI
 
 
+CONF = cfg.CONF
+CONF.import_group('amphora_agent', 'octavia.common.config')
+
 ETH_PORT_CONF = 'plug_vip_ethX.conf.j2'
 
 ETH_X_VIP_CONF = 'plug_port_ethX.conf.j2'
@@ -42,7 +49,7 @@ template_port = j2_env.get_template(ETH_X_VIP_CONF)
 template_vip = j2_env.get_template(ETH_PORT_CONF)
 
 
-def plug_vip(vip, subnet_cidr, gateway, mac_address):
+def plug_vip(vip, subnet_cidr, gateway, mac_address, vrrp_ip):
     # validate vip
     try:
         socket.inet_aton(vip)
@@ -72,14 +79,24 @@ def plug_vip(vip, subnet_cidr, gateway, mac_address):
     flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
     # mode 00644
     mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
-    with os.fdopen(os.open(name, flags, mode), 'w') as file:
+    with os.fdopen(os.open(name, flags, mode), 'w') as int_file:
-        file.write('auto lo\n')
+        int_file.write('auto lo\n')
-        file.write('iface lo inet loopback\n')
+        int_file.write('iface lo inet loopback\n')
-        file.write('source /etc/netns/{}/network/interfaces.d/*.cfg\n'.format(
+        if not CONF.amphora_agent.agent_server_network_file:
+            int_file.write('source /etc/netns/{}/network/'
+                           'interfaces.d/*.cfg\n'.format(
                                consts.AMPHORA_NAMESPACE))
 
     # write interface file
+
     mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+    # If we are using a consolidated interfaces file, just append
+    # otherwise clear the per interface file as we are rewriting it
+    # TODO(johnsom): We need a way to clean out old interfaces records
+    if CONF.amphora_agent.agent_server_network_file:
+        flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
+    else:
         flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
 
     with os.fdopen(os.open(interface_file_path, flags, mode),
@@ -89,7 +106,9 @@ def plug_vip(vip, subnet_cidr, gateway, mac_address):
             vip=vip,
             broadcast=broadcast,
             # assume for now only a fixed subnet size
-            netmask='255.255.255.0')
+            netmask='255.255.255.0',
+            gateway=gateway,
+            vrrp_ip=vrrp_ip)
         text_file.write(text)
 
     # Update the list of interfaces to add to the namespace
@@ -117,7 +136,7 @@ def plug_vip(vip, subnet_cidr, gateway, mac_address):
             vip=vip, interface=interface))), 202)
 
 
-def plug_network(mac_address):
+def plug_network(mac_address, fixed_ips):
     # This is the interface as it was initially plugged into the
     # default network namespace, this will likely always be eth1
     default_netns_interface = _interface_by_mac(mac_address)
@@ -138,14 +157,54 @@ def plug_network(mac_address):
     interface_file_path = util.get_network_interface_file(netns_interface)
 
     # write interface file
+
+    # If we are using a consolidated interfaces file, just append
+    # otherwise clear the per interface file as we are rewriting it
+    # TODO(johnsom): We need a way to clean out old interfaces records
+    if CONF.amphora_agent.agent_server_network_file:
+        flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
+    else:
         flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+
     # mode 00644
     mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
 
     with os.fdopen(os.open(interface_file_path, flags, mode),
                    'w') as text_file:
+        if fixed_ips is None:
             text = template_port.render(interface=netns_interface)
             text_file.write(text)
+        else:
+            ip_count = -1
+            for fixed_ip in fixed_ips:
+                if ip_count == -1:
+                    netns_ip_interface = netns_interface
+                    ip_count += 1
+                else:
+                    netns_ip_interface = "{int}:{ip}".format(
+                        int=netns_interface, ip=ip_count)
+                    ip_count += 1
+                try:
+                    ip_addr = fixed_ip['ip_address']
+                    cidr = fixed_ip['subnet_cidr']
+                    ip = ipaddress.ip_address(
+                        ip_addr if six.text_type == type(
+                            ip_addr) else six.u(ip_addr))
+                    network = ipaddress.ip_network(
+                        cidr if six.text_type == type(
+                            cidr) else six.u(cidr))
+                    broadcast = network.broadcast_address.exploded
+                    netmask = (network.prefixlen if ip.version is 6
+                               else network.netmask.exploded)
+                except ValueError:
+                    return flask.make_response(flask.jsonify(dict(
+                        message="Invalid network IP")), 400)
+                text = template_port.render(interface=netns_ip_interface,
+                                            ipv6=ip.version is 6,
+                                            ip_address=ip.exploded,
+                                            broadcast=broadcast,
+                                            netmask=netmask)
+                text_file.write(text)
 
     # Update the list of interfaces to add to the namespace
     _update_plugged_interfaces_file(netns_interface, mac_address)

octavia/amphorae/backends/agent/api_server/server.py

@@ -122,7 +122,8 @@ def plug_vip(vip):
     return plug.plug_vip(vip,
                          net_info['subnet_cidr'],
                          net_info['gateway'],
-                         net_info['mac_address'])
+                         net_info['mac_address'],
+                         net_info.get('vrrp_ip'))
 
 
 @app.route('/' + api_server.VERSION + '/plug/network', methods=['POST'])
@@ -133,7 +134,8 @@ def plug_network():
         assert 'mac_address' in port_info
     except Exception:
         raise exceptions.BadRequest(description='Invalid port information')
-    return plug.plug_network(port_info['mac_address'])
+    return plug.plug_network(port_info['mac_address'],
+                             port_info.get('fixed_ips'))
 
 
 @app.route('/' + api_server.VERSION + '/certificate', methods=['PUT'])

octavia/amphorae/backends/agent/api_server/templates/plug_port_ethX.conf.j2

@@ -14,5 +14,15 @@
 #    under the License.
 #}
 # Generated by Octavia agent
-auto {{ interface }} {{ interface }}:0
+auto {{ interface }}
+{%- if ip_address %}
+iface {{ interface }} inet{{ '6' if ipv6 }} static
+address {{ ip_address }}
+broadcast {{ broadcast }}
+netmask {{ netmask }}
+{%- else %}
 iface {{ interface }} inet dhcp
+auto {{ interface }}:0
+iface {{ interface }}:0 inet6 auto
+{%- endif %}
+

octavia/amphorae/backends/agent/api_server/templates/plug_vip_ethX.conf.j2

@@ -1,5 +1,6 @@
 {#
 # Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2016 Rackspace
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,7 +16,17 @@
 #}
 # Generated by Octavia agent
 auto {{ interface }} {{ interface }}:0
+
+{%- if vrrp_ip %}
+iface {{ interface }} inet static
+address {{ vrrp_ip }}
+broadcast {{ broadcast }}
+netmask {{ netmask }}
+gateway {{ gateway }}
+{%- else %}
 iface {{ interface }} inet dhcp
+{%- endif %}
+
 iface {{ interface }}:0 inet static
 address {{ vip }}
 broadcast {{ broadcast }}

octavia/amphorae/drivers/haproxy/rest_api_driver.py

@@ -123,16 +123,26 @@ class HaproxyAmphoraLoadBalancerDriver(
                 # tight coupling between the network driver and amphora
                 # driver.  We will need to revisit this to try and remove
                 # this tight coupling.
+                # NOTE (johnsom): I am loading the vrrp_ip into the
+                # net_info structure here so that I don't break
+                # compatibility with old amphora agent versions.
                 port = amphorae_network_config.get(amp.id).vrrp_port
                 net_info = {'subnet_cidr': subnet.cidr,
                             'gateway': subnet.gateway_ip,
-                            'mac_address': port.mac_address}
+                            'mac_address': port.mac_address,
+                            'vrrp_ip': amp.vrrp_ip}
                 self.client.plug_vip(amp,
                                      load_balancer.vip.ip_address,
                                      net_info)
 
     def post_network_plug(self, amphora, port):
-        port_info = {'mac_address': port.mac_address}
+        fixed_ips = []
+        for fixed_ip in port.fixed_ips:
+            ip = {'ip_address': fixed_ip.ip_address,
+                  'subnet_cidr': fixed_ip.subnet.cidr}
+            fixed_ips.append(ip)
+        port_info = {'mac_address': port.mac_address,
+                     'fixed_ips': fixed_ips}
         self.client.plug_network(amphora, port_info)
 
     def get_vrrp_interface(self, amphora):

octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py

@@ -552,13 +552,118 @@ class ServerTestCase(base.TestCase):
             handle = m()
             handle.write.assert_any_call(
                 '\n# Generated by Octavia agent\n'
-                'auto eth' + test_int_num + ' eth' + test_int_num +
+                'auto eth' + test_int_num +
-                ':0\niface eth' + test_int_num + ' inet dhcp')
+                '\niface eth' + test_int_num + ' inet dhcp\n'
+                'auto eth' + test_int_num + ':0\n'
+                'iface eth' + test_int_num + ':0 inet6 auto\n')
             mock_check_output.assert_called_with(
                 ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE,
                  'ifup', 'eth' + test_int_num], stderr=-2)
 
+        # fixed IPs happy path
+        port_info = {'mac_address': '123', 'fixed_ips': [
+            {'ip_address': '10.0.0.5', 'subnet_cidr': '10.0.0.0/24'}]}
+        mock_interfaces.side_effect = [['blah']]
+        mock_ifaddress.side_effect = [[netifaces.AF_LINK],
+                                      {netifaces.AF_LINK: [{'addr': '123'}]}]
+
+        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+        mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+        file_name = '/etc/netns/{0}/network/interfaces.d/eth{1}.cfg'.format(
+            consts.AMPHORA_NAMESPACE, test_int_num)
+        m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
+        with mock.patch('os.open') as mock_open, mock.patch.object(
+                os, 'fdopen', m) as mock_fdopen:
+            mock_open.return_value = 123
+            rv = self.app.post('/' + api_server.VERSION + "/plug/network",
+                               content_type='application/json',
+                               data=json.dumps(port_info))
+            self.assertEqual(202, rv.status_code)
+
+            mock_open.assert_any_call(file_name, flags, mode)
+            mock_fdopen.assert_any_call(123, 'w')
+
+            plug_inf_file = '/var/lib/octavia/plugged_interfaces'
+            flags = os.O_RDWR | os.O_CREAT
+            mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+            mock_open.assert_any_call(plug_inf_file, flags, mode)
+            mock_fdopen.assert_any_call(123, 'r+')
+
+            handle = m()
+            handle.write.assert_any_call(
+                '\n# Generated by Octavia agent\n'
+                'auto eth' + test_int_num +
+                '\niface eth' + test_int_num + ' inet static\n' +
+                'address 10.0.0.5\nbroadcast 10.0.0.255\n' +
+                'netmask 255.255.255.0\n')
+            mock_check_output.assert_called_with(
+                ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE,
+                 'ifup', 'eth' + test_int_num], stderr=-2)
+
+        # fixed IPs happy path IPv6
+        port_info = {'mac_address': '123', 'fixed_ips': [
+            {'ip_address': '2001:db8::2', 'subnet_cidr': '2001:db8::/32'}]}
+        mock_interfaces.side_effect = [['blah']]
+        mock_ifaddress.side_effect = [[netifaces.AF_LINK],
+                                      {netifaces.AF_LINK: [{'addr': '123'}]}]
+
+        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+        mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+        file_name = '/etc/netns/{0}/network/interfaces.d/eth{1}.cfg'.format(
+            consts.AMPHORA_NAMESPACE, test_int_num)
+        m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
+        with mock.patch('os.open') as mock_open, mock.patch.object(
+                os, 'fdopen', m) as mock_fdopen:
+            mock_open.return_value = 123
+            rv = self.app.post('/' + api_server.VERSION + "/plug/network",
+                               content_type='application/json',
+                               data=json.dumps(port_info))
+            self.assertEqual(202, rv.status_code)
+
+            mock_open.assert_any_call(file_name, flags, mode)
+            mock_fdopen.assert_any_call(123, 'w')
+
+            plug_inf_file = '/var/lib/octavia/plugged_interfaces'
+            flags = os.O_RDWR | os.O_CREAT
+            mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+            mock_open.assert_any_call(plug_inf_file, flags, mode)
+            mock_fdopen.assert_any_call(123, 'r+')
+
+            handle = m()
+            handle.write.assert_any_call(
+                '\n# Generated by Octavia agent\n'
+                'auto eth' + test_int_num +
+                '\niface eth' + test_int_num + ' inet6 static\n' +
+                'address 2001:0db8:0000:0000:0000:0000:0000:0002\n'
+                'broadcast 2001:0db8:ffff:ffff:ffff:ffff:ffff:ffff\n' +
+                'netmask 32\n')
+            mock_check_output.assert_called_with(
+                ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE,
+                 'ifup', 'eth' + test_int_num], stderr=-2)
+
+        # fixed IPs, bogus IP
+        port_info = {'mac_address': '123', 'fixed_ips': [
+            {'ip_address': '10005', 'subnet_cidr': '10.0.0.0/24'}]}
+        mock_interfaces.side_effect = [['blah']]
+        mock_ifaddress.side_effect = [[netifaces.AF_LINK],
+                                      {netifaces.AF_LINK: [{'addr': '123'}]}]
+
+        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+        mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+        file_name = '/etc/netns/{0}/network/interfaces.d/eth{1}.cfg'.format(
+            consts.AMPHORA_NAMESPACE, test_int_num)
+        m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
+        with mock.patch('os.open') as mock_open, mock.patch.object(
+                os, 'fdopen', m) as mock_fdopen:
+            mock_open.return_value = 123
+            rv = self.app.post('/' + api_server.VERSION + "/plug/network",
+                               content_type='application/json',
+                               data=json.dumps(port_info))
+            self.assertEqual(400, rv.status_code)
+
         # same as above but ifup fails
+        port_info = {'mac_address': '123', 'fixed_ips': [
+            {'ip_address': '10.0.0.5', 'subnet_cidr': '10.0.0.0/24'}]}
         mock_interfaces.side_effect = [['blah']]
         mock_ifaddress.side_effect = [[netifaces.AF_LINK],
                                       {netifaces.AF_LINK: [{'addr': '123'}]}]
@@ -654,7 +759,7 @@ class ServerTestCase(base.TestCase):
             handle.write.assert_any_call(
                 '\n# Generated by Octavia agent\n'
                 'auto blah blah:0\n'
-                'iface blah inet dhcp\n'
+                'iface blah inet dhcp\n\n'
                 'iface blah:0 inet static\n'
                 'address 203.0.113.2\n'
                 'broadcast 203.0.113.255\n'

octavia/tests/functional/amphorae/backend/agent/api_server/test_server_sysvinit.py

@@ -558,8 +558,10 @@ class ServerTestCase(base.TestCase):
             handle = m()
             handle.write.assert_any_call(
                 '\n# Generated by Octavia agent\n'
-                'auto eth' + test_int_num + ' eth' + test_int_num + ':0\n'
+                'auto eth' + test_int_num + '\n'
-                'iface eth' + test_int_num + ' inet dhcp')
+                'iface eth' + test_int_num + ' inet dhcp\n'
+                'auto eth' + test_int_num + ':0\n'
+                'iface eth' + test_int_num + ':0 inet6 auto\n')
             mock_check_output.assert_called_with(
                 ['ip', 'netns', 'exec', 'amphora-haproxy', 'ifup',
                  'eth' + test_int_num], stderr=-2)
@@ -660,7 +662,7 @@ class ServerTestCase(base.TestCase):
             handle.write.assert_any_call(
                 '\n# Generated by Octavia agent\n'
                 'auto blah blah:0\n'
-                'iface blah inet dhcp\n'
+                'iface blah inet dhcp\n\n'
                 'iface blah:0 inet static\n'
                 'address 203.0.113.2\n'
                 'broadcast 203.0.113.255\n'

octavia/tests/unit/amphorae/drivers/haproxy/test_rest_api_driver.py

@@ -32,11 +32,9 @@ FAKE_CIDR = '10.0.0.0/24'
 FAKE_GATEWAY = '10.0.0.1'
 FAKE_IP = 'fake'
 FAKE_PEM_FILENAME = "file_name"
-FAKE_SUBNET_INFO = {'subnet_cidr': FAKE_CIDR,
-                    'gateway': FAKE_GATEWAY,
-                    'mac_address': '123'}
 FAKE_UUID_1 = uuidutils.generate_uuid()
 FAKE_VRRP_IP = '10.1.0.1'
+FAKE_MAC_ADDRESS = '123'
 
 
 class TestHaproxyAmphoraLoadBalancerDriverTest(base.TestCase):
@@ -55,7 +53,16 @@ class TestHaproxyAmphoraLoadBalancerDriverTest(base.TestCase):
         self.amp = self.sl.load_balancer.amphorae[0]
         self.sv = sample_configs.sample_vip_tuple()
         self.lb = self.sl.load_balancer
-        self.port = network_models.Port(mac_address='123')
+        self.fixed_ip = mock.MagicMock()
+        self.fixed_ip.ip_address = '10.0.0.5'
+        self.fixed_ip.subnet.cidr = '10.0.0.0/24'
+        self.port = network_models.Port(mac_address=FAKE_MAC_ADDRESS,
+                                        fixed_ips=[self.fixed_ip])
+
+        self.subnet_info = {'subnet_cidr': FAKE_CIDR,
+                            'gateway': FAKE_GATEWAY,
+                            'mac_address': FAKE_MAC_ADDRESS,
+                            'vrrp_ip': self.amp.vrrp_ip}
 
     @mock.patch('octavia.common.tls_utils.cert_parser.load_certificates_data')
     @mock.patch('octavia.common.tls_utils.cert_parser.get_host_names')
@@ -154,12 +161,23 @@ class TestHaproxyAmphoraLoadBalancerDriverTest(base.TestCase):
         amphorae_network_config.get().vrrp_port = self.port
         self.driver.post_vip_plug(self.lb, amphorae_network_config)
         self.driver.client.plug_vip.assert_called_once_with(
-            self.amp, self.lb.vip.ip_address, FAKE_SUBNET_INFO)
+            self.amp, self.lb.vip.ip_address, self.subnet_info)
 
     def test_post_network_plug(self):
+        # Test dhcp path
+        port = network_models.Port(mac_address=FAKE_MAC_ADDRESS, fixed_ips=[])
+        self.driver.post_network_plug(self.amp, port)
+        self.driver.client.plug_network.assert_called_once_with(
+            self.amp, dict(mac_address=FAKE_MAC_ADDRESS, fixed_ips=[]))
+
+        self.driver.client.plug_network.reset_mock()
+
+        # Test fixed IP path
         self.driver.post_network_plug(self.amp, self.port)
         self.driver.client.plug_network.assert_called_once_with(
-            self.amp, dict(mac_address='123'))
+            self.amp, dict(mac_address=FAKE_MAC_ADDRESS,
+                           fixed_ips=[dict(ip_address='10.0.0.5',
+                                           subnet_cidr='10.0.0.0/24')]))
 
     def test_get_vrrp_interface(self):
         self.driver.get_vrrp_interface(self.amp)
@@ -174,11 +192,16 @@ class TestAmphoraAPIClientTest(base.TestCase):
         self.driver = driver.AmphoraAPIClient()
         self.base_url = "https://127.0.0.1:9443/0.5"
         self.amp = models.Amphora(lb_network_ip='127.0.0.1', compute_id='123')
-        self.port_info = dict(mac_address='123')
+        self.port_info = dict(mac_address=FAKE_MAC_ADDRESS)
         # Override with much lower values for testing purposes..
         conf = oslo_fixture.Config(cfg.CONF)
         conf.config(group="haproxy_amphora", connection_max_retries=2)
 
+        self.subnet_info = {'subnet_cidr': FAKE_CIDR,
+                            'gateway': FAKE_GATEWAY,
+                            'mac_address': FAKE_MAC_ADDRESS,
+                            'vrrp_ip': self.amp.vrrp_ip}
+
     def test_request(self):
         self.assertRaises(driver_except.TimeOutException,
                           self.driver.request,
@@ -691,7 +714,7 @@ class TestAmphoraAPIClientTest(base.TestCase):
         m.post("{base}/plug/vip/{vip}".format(
             base=self.base_url, vip=FAKE_IP)
         )
-        self.driver.plug_vip(self.amp, FAKE_IP, FAKE_SUBNET_INFO)
+        self.driver.plug_vip(self.amp, FAKE_IP, self.subnet_info)
         self.assertTrue(m.called)
 
     @requests_mock.mock()

releasenotes/notes/support-networks-without-dhcp-3458a063333ab7a8.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Adds support for networks that do not have DHCP services enabled.
+upgrade:
+  - To support networks without DHCP you must upgrade your amphora image.

requirements.txt

@@ -38,5 +38,5 @@ taskflow>=1.26.0 # Apache-2.0
 #for the amphora api
 Flask!=0.11,<1.0,>=0.10 # BSD
 netifaces>=0.10.4 # MIT
-
+ipaddress>=1.0.7;python_version<'3.3'  # PSF
 cryptography!=1.3.0,>=1.0 # BSD/Apache-2.0