Placement: utils

* to generate Placement trait names,
* to generate Placement resource provider UUIDs, and
* to uniformly parse and validate Placement-related config options in
  all agents.

Change-Id: I192d99673feba97a95af995923b266e2f8b58c6d
Needed-By: https://review.openstack.org/577223
Partial-Bug: #1578989
See-Also: https://review.openstack.org/502306 (nova spec)
See-Also: https://review.openstack.org/508149 (neutron spec)

lower-constraints.txt

@@ -37,6 +37,7 @@ netifaces==0.10.4
 openstackdocstheme==1.18.1
 os-api-ref==1.4.0
 os-client-config==1.28.0
+os-traits==0.9.0
 oslo.concurrency==3.26.0
 oslo.config==5.2.0
 oslo.context==2.19.2

neutron_lib/placement/utils.py

@@ -0,0 +1,243 @@
+# Copyright 2018 Ericsson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+import os_traits
+from oslo_log import log as logging
+import six
+
+from neutron_lib._i18n import _
+from neutron_lib import constants as const
+from neutron_lib.placement import constants as place_const
+
+
+LOG = logging.getLogger(__name__)
+
+
+def physnet_trait(physnet):
+    """A Placement trait name to represent being connected to a physnet.
+
+    :param physnet: The physnet name.
+    :returns: The trait name representing the physnet.
+    """
+    return os_traits.normalize_name('%s%s' % (
+        place_const.TRAIT_PREFIX_PHYSNET, physnet))
+
+
+def vnic_type_trait(vnic_type):
+    """A Placement trait name to represent support for a vnic_type.
+
+    :param physnet: The vnic_type.
+    :returns: The trait name representing the vnic_type.
+    """
+    return os_traits.normalize_name('%s%s' % (
+        place_const.TRAIT_PREFIX_VNIC_TYPE, vnic_type))
+
+
+def six_uuid5(namespace, name):
+    """A uuid.uuid5 variant that takes utf-8 'name' both in Python 2 and 3.
+
+    :param namespace: A UUID object used as a namespace in the generation of a
+                      v5 UUID.
+    :param name: Any string (either bytecode or unicode) used as a name in the
+                 generation of a v5 UUID.
+    :returns: A v5 UUID object.
+    """
+
+    # NOTE(bence romsics):
+    # uuid.uuid5() behaves seemingly consistently but still incompatibly
+    # different in cPython 2 and 3. Both expects the 'name' parameter to have
+    # the type of the default string literal in each language version.
+    # That is:
+    #     The cPython 2 variant expects a byte string.
+    #     The cPython 3 variant expects a unicode string.
+    # Which types are called respectively 'str' and 'str' for the sake of
+    # confusion. But the sha1() hash inside uuid5() always needs a byte string,
+    # so we have to treat the two versions asymmetrically. See also:
+    #
+    # cPython 2.7:
+    # https://github.com/python/cpython/blob
+    #        /ea9a0994cd0f4bd37799b045c34097eb21662b3d/Lib/uuid.py#L603
+    # cPython 3.6:
+    # https://github.com/python/cpython/blob
+    #        /e9e2fd75ccbc6e9a5221cf3525e39e9d042d843f/Lib/uuid.py#L628
+    if six.PY2:
+        name = name.encode('utf-8')
+    return uuid.uuid5(namespace=namespace, name=name)
+
+
+# NOTE(bence romsics): The spec said: "Agent resource providers shall
+# be identified by their already existing Neutron agent UUIDs [...]"
+#
+# https://review.openstack.org/#/c/508149/14/specs/rocky
+#        /minimum-bandwidth-allocation-placement-api.rst@465
+#
+# However we forgot that agent UUIDs are not stable through a few
+# admin operations like after a manual 'openstack network agent
+# delete'. Here we make up a stable UUID instead.
+def agent_resource_provider_uuid(namespace, host):
+    """Generate a stable UUID for an agent.
+
+    :param namespace: A UUID object identifying a mechanism driver (including
+                      its agent).
+    :param host: The hostname of the agent.
+    :returns: A unique and stable UUID identifying an agent.
+    """
+    return six_uuid5(namespace=namespace, name=host)
+
+
+def device_resource_provider_uuid(namespace, host, device, separator=':'):
+    """Generate a stable UUID for a physical network device.
+
+    :param namespace: A UUID object identifying a mechanism driver (including
+                      its agent).
+    :param host: The hostname of the agent.
+    :param device: A host-unique name of the physical network device.
+    :param separator: A string used in assembling a name for uuid5(). Choose
+                      one that cannot occur either in 'host' or 'device'.
+                      Optional.
+    :returns: A unique and stable UUID identifying a physical network device.
+    """
+    name = separator.join([host, device])
+    return six_uuid5(namespace=namespace, name=name)
+
+
+def _parse_bandwidth_value(bw_str):
+    """Parse the config string of a bandwidth value to an integer.
+
+    :param bw_str: A decimal string represantation of an integer, allowing
+                   the empty string.
+    :raises: ValueError on invalid input.
+    :returns: The bandwidth value as an integer or None if not set in config.
+    """
+    try:
+        bw = None
+        if bw_str:
+            bw = int(bw_str)
+            if bw < 0:
+                raise ValueError()
+    except ValueError:
+        raise ValueError(_(
+            'Cannot parse resource_provider_bandwidths. '
+            'Expected: non-negative integer bandwidth value, got: %s') %
+            bw_str)
+    return bw
+
+
+def parse_rp_bandwidths(bandwidths):
+    """Parse and validate config option: resource_provider_bandwidths.
+
+    Input in the config:
+        resource_provider_bandwidths = eth0:10000:10000,eth1::10000,eth2::,eth3
+    Input here:
+        ['eth0:10000:10000', 'eth1::10000', 'eth2::', 'eth3']
+    Output:
+        {
+            'eth0': {'egress': 10000, 'ingress': 10000},
+            'eth1': {'egress': None, 'ingress': 10000},
+            'eth2': {'egress': None, 'ingress': None},
+            'eth3': {'egress': None, 'ingress': None},
+        }
+
+    :param bandwidths: The list of 'interface:egress:ingress' bandwidth
+                       config options as pre-parsed by oslo_config.
+    :raises: ValueError on invalid input.
+    :returns: The fully parsed bandwidth config as a dict of dicts.
+    """
+
+    rv = {}
+    for bandwidth in bandwidths:
+        if ':' not in bandwidth:
+            bandwidth += '::'
+        try:
+            device, egress_str, ingress_str = bandwidth.split(':')
+        except ValueError:
+            raise ValueError(_(
+                'Cannot parse resource_provider_bandwidths. '
+                'Expected: DEVICE:EGRESS:INGRESS, got: %s') % bandwidth)
+        if device in rv:
+            raise ValueError(_(
+                'Cannot parse resource_provider_bandwidths. '
+                'Same device listed multiple times: %s') % device)
+        egress = _parse_bandwidth_value(egress_str)
+        ingress = _parse_bandwidth_value(ingress_str)
+        rv[device] = {
+            const.EGRESS_DIRECTION: egress,
+            const.INGRESS_DIRECTION: ingress,
+        }
+    return rv
+
+
+def parse_rp_inventory_defaults(inventory_defaults):
+    """Parse and validate config option: parse_rp_inventory_defaults.
+
+    Cast the dict values to the proper numerical types.
+
+    Input in the config:
+        resource_provider_inventory_defaults = allocation_ratio:1.0,min_unit:1
+    Input here:
+        {
+            'allocation_ratio': '1.0',
+            'min_unit': '1',
+        }
+    Output here:
+        {
+            'allocation_ratio': 1.0,
+            'min_unit': 1,
+        }
+
+    :param inventory_defaults: The dict of inventory parameters and values (as
+                               strings) as pre-parsed by oslo_config.
+    :raises: ValueError on invalid input.
+    :returns: The fully parsed inventory parameters and values (as numerical
+              values) as a dict.
+    """
+
+    unexpected_options = (set(inventory_defaults.keys()) -
+                          place_const.INVENTORY_OPTIONS)
+    if unexpected_options:
+        raise ValueError(_(
+            'Cannot parse inventory_defaults. Unexpected options: %s') %
+            ','.join(unexpected_options))
+
+    # allocation_ratio is a float
+    try:
+        if 'allocation_ratio' in inventory_defaults:
+            inventory_defaults['allocation_ratio'] = float(
+                inventory_defaults['allocation_ratio'])
+            if inventory_defaults['allocation_ratio'] < 0:
+                raise ValueError()
+    except ValueError:
+        raise ValueError(_(
+            'Cannot parse inventory_defaults.allocation_ratio. '
+            'Expected: non-negative float, got: %s') %
+            inventory_defaults['allocation_ratio'])
+
+    # the others are ints
+    for key in ('min_unit', 'max_unit', 'reserved', 'step_size'):
+        try:
+            if key in inventory_defaults:
+                inventory_defaults[key] = int(inventory_defaults[key])
+                if inventory_defaults[key] < 0:
+                    raise ValueError()
+        except ValueError:
+            raise ValueError(_(
+                'Cannot parse inventory_defaults.%(key)s. '
+                'Expected: non-negative int, got: %(got)s') % {
+                    'key': key,
+                    'got': inventory_defaults[key],
+            })
+
+    return inventory_defaults

neutron_lib/tests/unit/placement/test_utils.py

@@ -0,0 +1,179 @@
+# Copyright 2018 Ericsson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+from neutron_lib.placement import utils as place_utils
+from neutron_lib.tests import _base as base
+
+
+class TestPlacementUtils(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestPlacementUtils, self).setUp()
+
+        self._uuid_ns = uuid.UUID('94fedd4d-1ce0-4bb3-9c9a-c9c0f56de154')
+
+    def test_physnet_trait(self):
+        self.assertEqual(
+            'CUSTOM_PHYSNET_SOME_PHYSNET',
+            place_utils.physnet_trait('some-physnet'))
+
+    def test_vnic_type_trait(self):
+        self.assertEqual(
+            'CUSTOM_VNIC_TYPE_SOMEVNICTYPE',
+            place_utils.vnic_type_trait('somevnictype'))
+
+    def test_six_uuid5_literal(self):
+        try:
+            # assertNotRaises
+            place_utils.six_uuid5(
+                namespace=self._uuid_ns,
+                name='may or may not be a unicode string' +
+                     ' depending on Python version')
+        except Exception:
+            self.fail('could not generate uuid')
+
+    def test_six_uuid5_unicode(self):
+        try:
+            # assertNotRaises
+            place_utils.six_uuid5(
+                namespace=self._uuid_ns,
+                name=u'unicode string')
+        except Exception:
+            self.fail('could not generate uuid')
+
+    def test_agent_resource_provider_uuid(self):
+        try:
+            # assertNotRaises
+            place_utils.agent_resource_provider_uuid(
+                namespace=self._uuid_ns,
+                host='some host')
+        except Exception:
+            self.fail('could not generate agent resource provider uuid')
+
+    def test_device_resource_provider_uuid(self):
+        try:
+            # assertNotRaises
+            place_utils.device_resource_provider_uuid(
+                namespace=self._uuid_ns,
+                host='some host',
+                device='some device')
+        except Exception:
+            self.fail('could not generate device resource provider uuid')
+
+    def test_agent_resource_provider_uuid_stable(self):
+        uuid_a = place_utils.agent_resource_provider_uuid(
+            namespace=self._uuid_ns,
+            host='somehost')
+        uuid_b = place_utils.agent_resource_provider_uuid(
+            namespace=self._uuid_ns,
+            host='somehost')
+        self.assertEqual(uuid_a, uuid_b)
+
+    def test_device_resource_provider_uuid_stable(self):
+        uuid_a = place_utils.device_resource_provider_uuid(
+            namespace=self._uuid_ns,
+            host='somehost',
+            device='some-device')
+        uuid_b = place_utils.device_resource_provider_uuid(
+            namespace=self._uuid_ns,
+            host='somehost',
+            device='some-device')
+        self.assertEqual(uuid_a, uuid_b)
+
+    def test_parse_rp_bandwidths(self):
+        self.assertEqual(
+            {},
+            place_utils.parse_rp_bandwidths([]),
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': None, 'ingress': None}},
+            place_utils.parse_rp_bandwidths(['eth0']),
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': None, 'ingress': None}},
+            place_utils.parse_rp_bandwidths(['eth0::']),
+        )
+
+        self.assertRaises(
+            ValueError,
+            place_utils.parse_rp_bandwidths,
+            ['eth0::', 'eth0::'],
+        )
+
+        self.assertRaises(
+            ValueError,
+            place_utils.parse_rp_bandwidths,
+            ['eth0:not a number:not a number'],
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': 1, 'ingress': None}},
+            place_utils.parse_rp_bandwidths(['eth0:1:']),
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': None, 'ingress': 1}},
+            place_utils.parse_rp_bandwidths(['eth0::1']),
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': 1, 'ingress': 1}},
+            place_utils.parse_rp_bandwidths(['eth0:1:1']),
+        )
+
+        self.assertEqual(
+            {'eth0': {'egress': 1, 'ingress': 1},
+             'eth1': {'egress': 10, 'ingress': 10}},
+            place_utils.parse_rp_bandwidths(['eth0:1:1', 'eth1:10:10']),
+        )
+
+    def test_parse_rp_inventory_defaults(self):
+        self.assertEqual(
+            {},
+            place_utils.parse_rp_inventory_defaults({}),
+        )
+
+        self.assertRaises(
+            ValueError,
+            place_utils.parse_rp_inventory_defaults,
+            {'allocation_ratio': '-1.0'}
+        )
+
+        self.assertEqual(
+            {'allocation_ratio': 1.0},
+            place_utils.parse_rp_inventory_defaults(
+                {'allocation_ratio': '1.0'}),
+        )
+
+        self.assertRaises(
+            ValueError,
+            place_utils.parse_rp_inventory_defaults,
+            {'min_unit': '-1'}
+        )
+
+        self.assertEqual(
+            {'min_unit': 1},
+            place_utils.parse_rp_inventory_defaults(
+                {'min_unit': '1'}),
+        )
+
+        self.assertRaises(
+            ValueError,
+            place_utils.parse_rp_inventory_defaults,
+            {'no such inventory parameter': 1}
+        )

releasenotes/notes/placement-utils-a66e6b302d2bc8f0.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    neutron-lib now has a new module: ``neutron_lib.placement.utils``.
+    This module contains logic that is to be shared between in-tree
+    Neutron components and possibly out-of-tree Neutron agents that want
+    to support features involving the Placement service (for example
+    guaranteed minimum bandwidth).

requirements.txt

@@ -24,4 +24,5 @@ oslo.versionedobjects>=1.31.2 # Apache-2.0
 osprofiler>=1.4.0 # Apache-2.0
 WebOb>=1.7.1 # MIT
 weakrefmethod>=1.0.2;python_version=='2.7'  # PSF
+os-traits>=0.9.0 # Apache-2.0