python 3.5 support

Enabling python 3.5 support in light of the python 2.7 EOL and
in line with openstack's support of python 3.5. Not everything
has been tested and hence some things may not work as expected
on python 3.5.

Change-Id: I5f640416da409a70850d42b7800c879c79457542

networking_vpp/agent/server.py

@@ -22,9 +22,9 @@
 # that work was not repeated here where the aim was to get the APIs
 # worked out.  The two codebases will merge in the future.
 
+from __future__ import absolute_import
 # eventlet must be monkey patched early or we confuse urllib3.
 import eventlet
-import ipaddress
 
 # We actually need to co-operate with a threaded callback in VPP, so
 # don't monkey patch the thread operations.
@@ -35,15 +35,15 @@ from collections import defaultdict
 from collections import namedtuple
 import etcd
 import eventlet.semaphore
-from ipaddress import ip_address
-from ipaddress import ip_network
+import ipaddress
 import os
 import re
+import six
 import sys
 import time
-import vpp
 
 from networking_vpp._i18n import _
+from networking_vpp.agent import vpp
 from networking_vpp import compat
 from networking_vpp.compat import n_const
 from networking_vpp.compat import net_utils
@@ -111,6 +111,19 @@ def eventlet_lock(name):
         return func_wrap
     return eventlet_lock_decorator
 
+
+# TODO(onong): move to common file in phase 2
+def ipnet(ip):
+    return ipaddress.ip_network(six.text_type(ip))
+
+
+def ipaddr(ip):
+    return ipaddress.ip_address(six.text_type(ip))
+
+
+def ipint(ip):
+    return ipaddress.ip_interface(six.text_type(ip))
+
 ######################################################################
 
 # This mirrors functionality in Neutron so that we're creating a name
@@ -529,6 +542,7 @@ class VPPForwarder(object):
                         opens[file] = opens[file] + 1
 
             # Report on any sockets that are open exactly twice (VPP + KVM)
+            # (note list clone so that we can delete entries)
             for f in list(opens.keys()):
                 if opens[f] != 2:
                     del opens[f]
@@ -1475,6 +1489,8 @@ class VPPForwarder(object):
                       "to acl mapping. Possible reason: vpp "
                       "may have been restarted on host.")
 
+        # py3 note: in py3 keys() does not return a list but the following
+        # seems to work fine. Enclose in list() is problems arise.
         return self.secgroups.keys()
 
     def get_secgroup_acl_map(self):
@@ -1584,7 +1600,7 @@ class VPPForwarder(object):
 
         def _get_ip_version(ip):
             """Return the IP Version i.e. 4 or 6"""
-            return ip_network(unicode(ip)).version
+            return ipnet(ip).version
 
         def _get_ip_prefix_length(ip):
             """Return the IP prefix length value
@@ -1596,7 +1612,7 @@ class VPPForwarder(object):
             i.e. 32 if IPv4 and 128 if IPv6
             if "ip" is an ip_network return its prefix_length
             """
-            return ip_network(unicode(ip)).prefixlen
+            return ipnet(ip).prefixlen
 
         src_mac_mask = _pack_mac('FF:FF:FF:FF:FF:FF')
         mac_ip_rules = []
@@ -1712,7 +1728,7 @@ class VPPForwarder(object):
                  an IPv4 or IPv6 network with prefix_length e.g. 1.1.1.0/24
         """
         # Works for both addresses and the net address of masked networks
-        return ip_network(unicode(ip_addr)).network_address.packed
+        return ipnet(ip_addr).network_address.packed
 
     def _get_snat_indexes(self, floatingip_dict):
         """Return the internal and external interface indices for SNAT.
@@ -1883,7 +1899,7 @@ class VPPForwarder(object):
                 self.add_local_gpe_mapping(seg_id, loopback_mac)
         # Set the gateway IP address on the BVI interface, if not already set
         addresses = self.vpp.get_interface_ip_addresses(loopback_idx)
-        gw_ip_obj = ipaddress.ip_address(unicode(gateway_ip))
+        gw_ip_obj = ipaddr(gateway_ip)
         for address in addresses:
             if address[0] == gw_ip_obj:
                 break
@@ -1960,8 +1976,7 @@ class VPPForwarder(object):
 
     def _get_ip_network(self, gateway_ip, prefixlen):
         """Returns the IP network for the gateway in CIDR form."""
-        return str(ipaddress.ip_interface(unicode(gateway_ip + "/"
-                                          + str(prefixlen))).network)
+        return str(ipint(gateway_ip + "/" + str(prefixlen)).network)
 
     def default_route_in_default_vrf(self, router_dict, is_add=True):
         # ensure that default route in default VRF is present
@@ -2416,7 +2431,7 @@ class VPPForwarder(object):
         lset_mapping = self.gpe_map[gpe_lset_name]
         if (mac, vni) not in self.gpe_map['remote_map'] and mac not in \
                 lset_mapping['local_map']:
-            is_ip4 = 1 if ip_network(unicode(remote_ip)).version == 4 else 0
+            is_ip4 = 1 if ipnet(remote_ip).version == 4 else 0
             remote_locator = {"is_ip4": is_ip4,
                               "priority": 1,
                               "weight": 1,
@@ -2427,8 +2442,8 @@ class VPPForwarder(object):
             # Add a LISP ARP/NDP entry for the remote VM's IPv4/v6 address.
             # If an ARP or NDP entry exists in the BD, replace it.
             bridge_domain = self.bridge_idx_for_lisp_segment(vni)
-            if ip and ip_network(unicode(ip)).version == 4:
-                int_ip = int(ip_address(unicode(ip)))
+            if ip and ipnet(ip).version == 4:
+                int_ip = int(ipaddr(ip))
                 if not self.vpp.exists_lisp_arp_entry(bridge_domain, int_ip):
                     self.vpp.add_lisp_arp_entry(mac,
                                                 bridge_domain,
@@ -2456,8 +2471,8 @@ class VPPForwarder(object):
             # Delete the LISP ARP entry for remote instance's IPv4 address
             # if it's present and the IP address is present
             bridge_domain = self.bridge_idx_for_lisp_segment(vni)
-            if ip and ip_network(unicode(ip)).version == 4:
-                int_ip = int(ip_address(unicode(ip)))
+            if ip and ipnet(ip).version == 4:
+                int_ip = int(ipaddr(ip))
                 if self.vpp.exists_lisp_arp_entry(bridge_domain, int_ip):
                     self.vpp.del_lisp_arp_entry(mac,
                                                 bridge_domain,
@@ -2538,13 +2553,12 @@ class VPPForwarder(object):
          self.gpe_underlay_mask) = self.gpe_src_cidr.split('/')
         physnet_ip_addrs = self.vpp.get_interface_ip_addresses(if_physnet)
         LOG.debug('Exising IP addresses %s', str(physnet_ip_addrs))
-        cidr = (ip_address(unicode(self.gpe_underlay_addr)),
+        cidr = (ipaddr(self.gpe_underlay_addr),
                 int(self.gpe_underlay_mask))
         if cidr not in physnet_ip_addrs:
             self.vpp.set_interface_address(
                 sw_if_index=if_physnet,
-                is_ipv6=1 if ip_network(unicode(self.gpe_underlay_addr)
-                                        ).version == 6 else 0,
+                is_ipv6=1 if ipnet(self.gpe_underlay_addr).version == 6 else 0,
                 address_length=int(self.gpe_underlay_mask),
                 address=self._pack_address(self.gpe_underlay_addr)
                 )
@@ -2926,7 +2940,7 @@ class EtcdListener(object):
         def _secgroup_rule(r):
             # Create a rule for the remote_ip_prefix (CIDR) value
             if r['remote_ip_addr']:
-                remote_ip_prefixes = [(unicode(r['remote_ip_addr']),
+                remote_ip_prefixes = [(six.text_type(r['remote_ip_addr']),
                                        r['ip_prefix_len'])]
             # Create a rule for each ip address in the remote_group
             else:
@@ -2939,10 +2953,10 @@ class EtcdListener(object):
                 # remote-group etcd watch event
                 self.vppf.remote_group_secgroups[remote_group].add(secgroup)
                 remote_ip_prefixes = [
-                    (unicode(ip), prefix_length) for port in
+                    (six.text_type(ip), prefix_length) for port in
                     self.vppf.remote_group_ports[remote_group]
                     for ip in self.vppf.port_ips[port]
-                    if ip_network(unicode(ip)).version == ip_version]
+                    if ipnet(ip).version == ip_version]
                 LOG.debug("remote_group: vppf.remote_group_ports:%s",
                           self.vppf.remote_group_ports
                           )
@@ -2954,7 +2968,7 @@ class EtcdListener(object):
                           self.vppf.remote_group_secgroups)
             # VPP API requires the IP addresses to be represented in binary
             rules = [SecurityGroupRule(r['is_ipv6'],
-                                       ip_address(ip_addr).packed,
+                                       ipaddr(ip_addr).packed,
                                        ip_prefix_len,
                                        r.get('remote_group_id', None),
                                        r['protocol'],
@@ -3361,6 +3375,8 @@ class EtcdListener(object):
 
         etcd_helper.clear_state(self.state_key_space)
 
+        # py3 note: in py3 keys() does not return a list but the following
+        # seems to work fine. Enclose in list() is problems arise.
         physnets = self.physnets.keys()
         etcd_helper.clear_state(self.physnet_key_space)
         for f in physnets:
@@ -3993,7 +4009,8 @@ def main():
 
     # Convert to the minutes unit that VPP uses:
     # (we round *up*)
-    mac_age_min = int((cfg.CONF.ml2_vpp.mac_age + 59) / 60)
+    # py3 note: using // since we want integer division
+    mac_age_min = int((cfg.CONF.ml2_vpp.mac_age + 59) // 60)
     vppf = VPPForwarder(physnets,
                         mac_age=mac_age_min,
                         vpp_cmd_queue_len=cfg.CONF.ml2_vpp.vpp_cmd_queue_len,

networking_vpp/agent/vpp.py

@@ -14,6 +14,8 @@
 #    under the License.
 
 
+from __future__ import absolute_import
+from __future__ import print_function
 import collections
 import enum
 import fnmatch
@@ -21,6 +23,7 @@ import grp
 import ipaddress
 import os
 import pwd
+import six
 import sys
 from threading import Lock
 import vpp_papi
@@ -31,12 +34,40 @@ L2_VTR_DISABLED = 0
 NO_BVI_SET = 4294967295
 
 
+def binary_type(s):
+    """Wrapper function to convert input string to bytes
+
+    TODO(onong): move to a common file in phase 2
+    """
+    return s.encode('ascii')
+
+
 def mac_to_bytes(mac):
-    return str(''.join(chr(int(x, base=16)) for x in mac.split(':')))
+    # py3 note:
+    # TODO(onong): PAPI has introduced a new macaddress object which seemingly
+    # takes care of conversion to/from MAC addr to string.
+    # TODO(onong): move to common file in phase 2
+    if six.PY2:
+        return ''.join(chr(int(x, base=16)) for x in mac.split(':'))
+    else:
+        return bytes.fromhex(mac.replace(':', ''))
 
 
 def fix_string(s):
-    return s.rstrip("\0").decode(encoding='ascii')
+    # py3 note:
+    # This function chops off any trailing NUL chars/bytes from strings that
+    # we get from VPP. Now, in case of py2, str and bytes are the same but
+    # there's a strict distinction between the two in py3. The code ensures
+    # that within the ML2 agent we follow the dictum of always dealing with
+    # strings and this function acts as the boundary where the conversion to
+    # string happens.
+    #
+    # TODO(onong): watch out for the upcoming PAPI change which introduces a
+    # string type for printable strings, so no longer the need for the funny
+    # chopping off of 0's at the end. But this function will still act as the
+    # boundary at which input is converted to string type.
+    # TODO(onong): move to common file in phase 2
+    return s.decode('ascii').rstrip('\0')
 
 
 def bytes_to_mac(mbytes):
@@ -111,16 +142,17 @@ class VPPInterface(object):
         VPP papi does not allow to set interface tag
         on interface creation for subinterface or loopback).
         """
-        # TODO(ijw): this is a race condition.
+        # TODO(ijw): this is a race condition - we should create the
+        # interface with a tag.
         self.call_vpp('sw_interface_tag_add_del',
                       is_add=1,
                       sw_if_index=if_idx,
-                      tag=str(tag))
+                      tag=binary_type(tag))
 
     def get_version(self):
         t = self.call_vpp('show_version')
 
-        return fix_string(t.version)
+        return t.version
 
     def semver(self):
         """Return the 'semantic' version components of a VPP version"""
@@ -148,8 +180,6 @@ class VPPInterface(object):
     ########################################
 
     def create_tap(self, ifname, mac=None, tag=""):
-        # (we don't like unicode in VPP hence str(ifname))
-
         if mac is not None:
             mac_bytes = mac_to_bytes(mac)
             use_random_mac = False
@@ -161,7 +191,7 @@ class VPPInterface(object):
                           use_random_mac=use_random_mac,
                           mac_address=mac_bytes,
                           host_if_name_set=True,
-                          host_if_name=str(ifname),
+                          host_if_name=binary_type(ifname),
                           id=0xffffffff,  # choose ifidx automatically
                           host_ip4_addr_set=False,
                           host_ip6_addr_set=False,
@@ -170,7 +200,7 @@ class VPPInterface(object):
                           host_mac_addr_set=False,
                           tx_ring_sz=1024,
                           rx_ring_sz=1024,
-                          tag=tag)
+                          tag=binary_type(tag))
 
         return t.sw_if_index  # will be -1 on failure (e.g. 'already exists')
 
@@ -196,12 +226,12 @@ class VPPInterface(object):
                          qemu_user=None, qemu_group=None, is_server=False):
         t = self.call_vpp('create_vhost_user_if',
                           is_server=is_server,
-                          sock_filename=str(ifpath),
+                          sock_filename=binary_type(ifpath),
                           renumber=False,
                           custom_dev_instance=0,
                           use_custom_mac=True,
                           mac_address=mac_to_bytes(mac),
-                          tag=tag)
+                          tag=binary_type(tag))
 
         if is_server:
             # The permission that qemu runs as.
@@ -468,7 +498,7 @@ class VPPInterface(object):
     def acl_add_replace(self, acl_index, tag, rules, count):
         t = self.call_vpp('acl_add_replace',
                           acl_index=acl_index,
-                          tag=str(tag),
+                          tag=binary_type(tag),
                           r=rules,
                           count=count)
         return t.acl_index
@@ -702,10 +732,10 @@ class VPPInterface(object):
         if not self.route_in_vrf(vrf, ip_address, prefixlen,
                                  next_hop_address, next_hop_sw_if_index,
                                  is_ipv6):
-            ip = ipaddress.ip_address(unicode(bytes_to_ip(ip_address,
-                                                          is_ipv6)))
+            ip = ipaddress.ip_address(six.text_type(bytes_to_ip(ip_address,
+                                                    is_ipv6)))
             if next_hop_address is not None:
-                next_hop = ipaddress.ip_address(unicode(bytes_to_ip(
+                next_hop = ipaddress.ip_address(six.text_type(bytes_to_ip(
                     next_hop_address, is_ipv6)))
                 self.LOG.debug('Adding route %s/%s to %s in router vrf:%s',
                                ip, prefixlen, next_hop, vrf)
@@ -744,10 +774,10 @@ class VPPInterface(object):
         if self.route_in_vrf(vrf, ip_address, prefixlen,
                              next_hop_address, next_hop_sw_if_index,
                              is_ipv6):
-            ip = ipaddress.ip_address(unicode(bytes_to_ip(ip_address,
-                                                          is_ipv6)))
+            ip = ipaddress.ip_address(six.text_type(bytes_to_ip(ip_address,
+                                                    is_ipv6)))
             if next_hop_address is not None:
-                next_hop = ipaddress.ip_address(unicode(bytes_to_ip(
+                next_hop = ipaddress.ip_address(six.text_type(bytes_to_ip(
                     next_hop_address, is_ipv6)))
                 self.LOG.debug('Deleting route %s/%s to %s in router vrf:%s',
                                ip, prefixlen, next_hop, vrf)
@@ -788,10 +818,10 @@ class VPPInterface(object):
         # in the VRF table by checking the ip_address, prefixlen and
         # Convert the ip & next_hop addresses to an ipaddress format for
         # comparison
-        ip = ipaddress.ip_address(unicode(bytes_to_ip(ip_address,
-                                                      is_ipv6)))
+        ip = ipaddress.ip_address(six.text_type(bytes_to_ip(ip_address,
+                                                is_ipv6)))
         if next_hop_address is not None:
-            next_hop = ipaddress.ip_address(unicode(
+            next_hop = ipaddress.ip_address(six.text_type(
                 bytes_to_ip(next_hop_address, is_ipv6)))
         else:
             next_hop = next_hop_address
@@ -803,13 +833,13 @@ class VPPInterface(object):
                 route.address_length == prefixlen and
                 # check if route.address == ip
                 ipaddress.ip_address(
-                    unicode(bytes_to_ip(route.address,
-                                        is_ipv6))) == ip and
+                    six.text_type(bytes_to_ip(route.address,
+                                  is_ipv6))) == ip and
                 # check if the next_hop is present the list
                 # of next hops in the route's path
                 next_hop in [ipaddress.ip_address(
-                    unicode(bytes_to_ip(p.next_hop,
-                                        is_ipv6))) for p in route.path]):
+                    six.text_type(bytes_to_ip(p.next_hop,
+                                  is_ipv6))) for p in route.path]):
                 self.LOG.debug('Route: %s/%s to %s exists in VRF:%s',
                                ip, prefixlen, next_hop, vrf)
                 return True
@@ -817,8 +847,8 @@ class VPPInterface(object):
                   route.address_length == prefixlen and
                   # check if route.address == ip
                   ipaddress.ip_address(
-                      unicode(bytes_to_ip(route.address,
-                                          is_ipv6))) == ip and
+                      six.text_type(bytes_to_ip(route.address,
+                                    is_ipv6))) == ip and
                   # check if the next_hop matches
                   sw_if_index in [p.sw_if_index for p in route.path]):
 

networking_vpp/services/l3_router/l3_vpp.py

@@ -14,6 +14,7 @@
 #  under the License.
 #
 
+from __future__ import absolute_import
 from ipaddress import ip_network
 from networking_vpp import constants as nvpp_const
 from oslo_config import cfg
@@ -26,6 +27,7 @@ from neutron.db import l3_gwmode_db
 from networking_vpp.compat import n_const as constants
 from networking_vpp.compat import n_exc
 from networking_vpp.compat import n_provider as provider
+import six
 
 # TODO(ijw): backward compatibility doesn't really belong here
 try:
@@ -160,7 +162,7 @@ class VppL3RouterPlugin(common_db_mixin.CommonDbMixin,
         router_dict['subnet_id'] = subnet['id']
         router_dict['fixed_ips'] = fixed_ips
 
-        address = ip_network(unicode(subnet['cidr']))
+        address = ip_network(six.text_type(subnet['cidr']))
         router_dict['network_id'] = network['id']
         router_dict['is_ipv6'] = True if address.version == 6 else False
         router_dict['prefixlen'] = address.prefixlen
@@ -236,7 +238,7 @@ class VppL3RouterPlugin(common_db_mixin.CommonDbMixin,
             subnet_id = fixed_ip['subnet_id']
             subnet = self._core_plugin.get_subnet(
                 context.elevated(), subnet_id)
-            address = ip_network(unicode(subnet['cidr']))
+            address = ip_network(six.text_type(subnet['cidr']))
             is_ipv6 = True if address.version == 6 else False
             gateways.append((fixed_ip['ip_address'],
                              address.prefixlen,

networking_vpp/utils/device_monitor.py

@@ -13,8 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import absolute_import
 import logging
 import os
+import six
 import socket
 import struct
 
@@ -309,6 +311,12 @@ class DeviceMonitor(object):
                         if attr_type == IFLA.IFNAME:
                             # As returned, includes a C-style \0
                             link_name = attr_body[:-1]
+                            # py3 note:
+                            # link_name is a bytes object so explicitly convert
+                            # to string in case of py3 otherwise we get an
+                            # exception.
+                            if six.PY3:
+                                link_name = link_name.decode('ascii')
                             break
 
                     if link_name is None:

requirements.txt

@@ -18,3 +18,4 @@ PyJWT
 cryptography>=2.0,<3
 pyOpenSSL
 pyinotify
+six

setup.cfg

@@ -15,6 +15,8 @@ classifier =
     Programming Language :: Python
     Programming Language :: Python :: 2
     Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.5
 
 [files]
 packages =