Introduce microversion

This patch aims at support microversion in Cyborg API.
The work items is the following:
1. remove old api_version_request.py to avoid the redundancy.
2. add a header in every API request and support to parse it and
route to the correct API method.
3. depracate v1 API.
4. add related UT.

For testing:

CTYPE="Content-Type: application/json"
AUTH="X-Auth-Token: $(openstack token issue -c id -f value)"

curl -s -H "$CTYPE" -H "$AUTH" -H "OpenStack-API-Version: accelerator
2.0" http://localhost/accelerator/devices

This will execute sucessfully because 2.0 microversion is supported.

curl -s -H "$CTYPE" -H "$AUTH" -H "OpenStack-API-Version: accelerator
latest" http://localhost/accelerator/devices

This will execute sucessfully because "latest" will be parsed to 2.0.

curl -s -H "$CTYPE" -H "$AUTH" -H "OpenStack-API-Version: accelerator
2.99" http://localhost/accelerator/devices

This will failed because we don't support 2.99 microversion.

Change-Id: Id9c34dc134d59b2332cefbcae5bbd7e6632e970d

cyborg/api/controllers/base.py

@@ -14,14 +14,19 @@
 #    under the License.
 
 import datetime
+import functools
 
 import inspect
+import microversion_parse
+
 import pecan
 from pecan import rest
+from webob import exc
 import wsme
 from wsme import types as wtypes
 
 API_V2 = 'v2'
+# name of attribute to keep version method information
 
 
 class APIBase(wtypes.Base):
@@ -62,3 +67,78 @@ class CyborgController(rest.RestController):
             return controller, remainder
 
         pecan.abort(405)
+
+
+@functools.total_ordering
+class Version(object):
+    """API Version object."""
+
+    current_api_version = 'OpenStack-API-Version'
+    """HTTP Header string carrying the requested version"""
+
+    min_api_version = 'OpenStack-API-Minimum-Version'
+    """HTTP response header"""
+
+    max_api_version = 'OpenStack-API-Maximum-Version'
+    """HTTP response header"""
+
+    def __init__(self, headers, default_version, latest_version):
+        """Create an API Version object from the supplied headers.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :raises: webob.HTTPNotAcceptable
+
+        """
+        (self.major, self.minor) = Version.parse_headers(
+            headers, default_version, latest_version)
+
+    def __repr__(self):
+        return '%s.%s' % (self.major, self.minor)
+
+    @staticmethod
+    def parse_headers(headers, default_version, latest_version):
+        """Determine the API version requested based on the headers supplied.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :returns: a tuple of (major, minor) version numbers
+        :raises: webob.HTTPNotAcceptable
+
+        """
+        version_str = microversion_parse.get_version(
+            headers,
+            service_type='accelerator')
+
+        minimal_version = (2, 0)
+
+        if version_str is None:
+            # If requested header is wrong, Cyborg answers with the minimal
+            # supported version.
+            return minimal_version
+
+        if version_str.lower() == 'latest':
+            parse_str = latest_version
+        else:
+            parse_str = version_str
+
+        try:
+            version = tuple(int(i) for i in parse_str.split('.'))
+        except ValueError:
+            version = minimal_version
+
+        if len(version) != 2:
+            raise exc.HTTPNotAcceptable(
+                "Invalid value for %s header" % Version.current_api_version)
+        return version
+
+    def __gt__(self, other):
+        return (self.major, self.minor) > (other.major, other.minor)
+
+    def __eq__(self, other):
+        return (self.major, self.minor) == (other.major, other.minor)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)

cyborg/api/controllers/root.py

@@ -13,15 +13,60 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import importlib
 import pecan
 from pecan import rest
 from wsme import types as wtypes
 
 from cyborg.api.controllers import base
+from cyborg.api.controllers import link
 from cyborg.api.controllers import v2
 from cyborg.api import expose
 
 
+class APIStatus(object):
+    CURRENT = "CURRENT"
+    SUPPORTED = "SUPPORTED"
+    DEPRECATED = "DEPRECATED"
+    EXPERIMENTAL = "EXPERIMENTAL"
+
+
+class Version(base.APIBase):
+    """An API version representation."""
+
+    id = wtypes.text
+    """The ID of the version, also acts as the release number"""
+
+    status = wtypes.text
+    """The state of this API version"""
+
+    max_version = wtypes.text
+    """The maximum version supported"""
+
+    min_version = wtypes.text
+    """The minimum version supported"""
+
+    links = [link.Link]
+    """A Link that points to a specific version of the API"""
+
+    @staticmethod
+    def convert(id, status=APIStatus.CURRENT):
+        version = Version()
+        if id == "v1":
+            version.max_version = None
+            version.min_version = None
+        else:
+            v = importlib.import_module(
+                'cyborg.api.controllers.%s.versions' % id)
+            version.max_version = v.max_version_string()
+            version.min_version = v.min_version_string()
+        version.id = id
+        version.status = status
+        version.links = [link.Link.make_link('self', pecan.request.host_url,
+                                             id, '', bookmark=True)]
+        return version
+
+
 class Root(base.APIBase):
     name = wtypes.text
     """The name of the API"""
@@ -29,17 +74,22 @@ class Root(base.APIBase):
     description = wtypes.text
     """Some information about this API"""
 
+    versions = [Version]
+    """Links to all the versions available in this API"""
+
+    default_version = Version
+    """A link to the default version of the API"""
+
     @staticmethod
     def convert():
         root = Root()
         root.name = 'OpenStack Cyborg API'
         root.description = (
-            'Cyborg (previously known as Nomad) is an '
+            "Cyborg is the OpenStack project for lifecycle "
-            'OpenStack project that aims to provide a general '
+            "management of hardware accelerators, such as GPUs,"
-            'purpose management framework for acceleration '
+            "FPGAs, AI chips, security accelerators, etc.")
-            'resources (i.e. various types of accelerators '
+        root.versions = [Version.convert('v2')]
-            'such as Crypto cards, GPU, FPGA, NVMe/NOF SSDs, '
+        root.default_version = Version.convert('v2')
-            'ODP, DPDK/SPDK and so on).')
         return root
 
 

cyborg/api/controllers/v2/__init__.py

@@ -17,16 +17,33 @@
 
 import pecan
 from pecan import rest
+from webob import exc
 from wsme import types as wtypes
 
+from cyborg.api import expose
+
 from cyborg.api.controllers import base
 from cyborg.api.controllers import link
-from cyborg.api.controllers.v2 import api_version_request
 from cyborg.api.controllers.v2 import arqs
 from cyborg.api.controllers.v2 import deployables
 from cyborg.api.controllers.v2 import device_profiles
 from cyborg.api.controllers.v2 import devices
-from cyborg.api import expose
+
+from cyborg.api.controllers.v2 import versions
+
+
+def min_version():
+    return base.Version(
+        {base.Version.current_api_version: ' '.join(
+            [versions.service_type_string(), versions.min_version_string()])},
+        versions.min_version_string(), versions.max_version_string())
+
+
+def max_version():
+    return base.Version(
+        {base.Version.current_api_version: ' '.join(
+            [versions.service_type_string(), versions.max_version_string()])},
+        versions.min_version_string(), versions.max_version_string())
 
 
 class V2(base.APIBase):
@@ -51,8 +68,8 @@ class V2(base.APIBase):
     def convert():
         v2 = V2()
         v2.id = 'v2.0'
-        v2.max_version = api_version_request.max_api_version().get_string()
+        v2.max_version = str(max_version())
-        v2.min_version = api_version_request.min_api_version().get_string()
+        v2.min_version = str(min_version())
         v2.status = 'CURRENT'
         v2.links = [
             link.Link.make_link('self', pecan.request.public_url,
@@ -73,5 +90,50 @@ class Controller(rest.RestController):
     def get(self):
         return V2.convert()
 
+    def _check_version(self, version, headers=None):
+        if headers is None:
+            headers = {}
+        # ensure that major version in the URL matches the header
+        if version.major != versions.BASE_VERSION:
+            raise exc.HTTPNotAcceptable(
+                "Mutually exclusive versions requested. Version %(ver)s "
+                "requested but not supported by this service. The supported "
+                "version range is: [%(min)s, %(max)s]." %
+                {'ver': version, 'min': versions.min_version_string(),
+                 'max': versions.max_version_string()},
+                headers=headers)
+        # ensure the minor version is within the supported range
+        if version < min_version() or version > max_version():
+            raise exc.HTTPNotAcceptable(
+                "Version %(ver)s was requested but the minor version is not "
+                "supported by this service. The supported version range is: "
+                "[%(min)s, %(max)s]." %
+                {'ver': version, 'min': versions.min_version_string(),
+                 'max': versions.max_version_string()},
+                headers=headers)
+
+    @pecan.expose()
+    def _route(self, args, request=None):
+        v = base.Version(pecan.request.headers, versions.min_version_string(),
+                         versions.max_version_string())
+
+        # The Vary header is used as a hint to caching proxies and user agents
+        # that the response is also dependent on the OpenStack-API-Version and
+        # not just the body and query parameters. See RFC 7231 for details.
+        pecan.response.headers['Vary'] = base.Version.current_api_version
+
+        # Always set the min and max headers
+        pecan.response.headers[base.Version.min_api_version] = (
+            versions.min_version_string())
+        pecan.response.headers[base.Version.max_api_version] = (
+            versions.max_version_string())
+
+        # assert that requested version is supported
+        self._check_version(v, pecan.response.headers)
+        pecan.response.headers[base.Version.current_api_version] = str(v)
+        pecan.request.version = v
+
+        return super(Controller, self)._route(args, request)
+
 
 __all__ = ('Controller',)

cyborg/api/controllers/v2/api_version_request.py

@@ -1,181 +0,0 @@
-# Copyright 2019 Intel, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import re
-
-from cyborg.common import exception
-
-# Define the minimum and maximum version of the API across all of the
-# REST API. The format of the version is:
-# X.Y where:
-#
-# - X will only be changed if a significant backwards incompatible API
-# change is made which affects the API as whole. That is, something
-# that is only very very rarely incremented.
-#
-# - Y when you make any change to the API. Note that this includes
-# semantic changes which may not affect the input or output formats or
-# even originate in the API code layer. We are not distinguishing
-# between backwards compatible and backwards incompatible changes in
-# the versioning system. It must be made clear in the documentation as
-# to what is a backwards compatible change and what is a backwards
-# incompatible one.
-
-#
-# You must update the API version history string below with a one or
-# two line description as well as update rest_api_version_history.rst
-REST_API_VERSION_HISTORY = """REST API Version History:
-
-    * 2.0 - Initial version.
-"""
-
-# The minimum and maximum versions of the API supported
-# The default api version request is defined to be the
-# minimum version of the API supported.
-# Note(cyeoh): This only applies for the v2.1 API once microversions
-# support is fully merged. It does not affect the V2 API.
-_MIN_API_VERSION = "2.0"
-_MAX_API_VERSION = "2.0"
-DEFAULT_API_VERSION = _MIN_API_VERSION
-
-
-# NOTE(Sundar): min and max versions declared as functions so we can
-# mock them for unittests. Do not use the constants directly anywhere
-# else.
-def min_api_version():
-    return APIVersionRequest(_MIN_API_VERSION)
-
-
-def max_api_version():
-    return APIVersionRequest(_MAX_API_VERSION)
-
-
-def is_supported(req, min_version=_MIN_API_VERSION,
-                 max_version=_MAX_API_VERSION):
-    """Check if API request version satisfies version restrictions.
-
-    :param req: request object
-    :param min_version: minimal version of API needed for correct
-           request processing
-    :param max_version: maximum version of API needed for correct
-           request processing
-
-    :returns: True if request satisfies minimal and maximum API version
-             requirements. False in other case.
-    """
-
-    return (APIVersionRequest(max_version) >= req.api_version_request >=
-            APIVersionRequest(min_version))
-
-
-class APIVersionRequest(object):
-    """This class represents an API Version Request with convenience
-    methods for manipulation and comparison of version
-    numbers that we need to do to implement microversions.
-    """
-
-    def __init__(self, version_string=None):
-        """Create an API version request object.
-
-        :param version_string: String representation of APIVersionRequest.
-            Correct format is 'X.Y', where 'X' and 'Y' are int values.
-            None value should be used to create Null APIVersionRequest,
-            which is equal to 0.0
-        """
-        self.ver_major = 0
-        self.ver_minor = 0
-
-        if version_string is not None:
-            match = re.match(r"^([1-9]\d*)\.([1-9]\d*|0)$",
-                             version_string)
-            if match:
-                self.ver_major = int(match.group(1))
-                self.ver_minor = int(match.group(2))
-            else:
-                raise exception.InvalidAPIVersionString(version=version_string)
-
-    def __str__(self):
-        """Debug/Logging representation of object."""
-        return ("API Version Request Major: %s, Minor: %s"
-                % (self.ver_major, self.ver_minor))
-
-    def is_null(self):
-        return self.ver_major == 0 and self.ver_minor == 0
-
-    def _format_type_error(self, other):
-        return TypeError("'%(other)s' should be an instance of '%(cls)s'" %
-                         {"other": other, "cls": self.__class__})
-
-    def __lt__(self, other):
-        if not isinstance(other, APIVersionRequest):
-            raise self._format_type_error(other)
-
-        return ((self.ver_major, self.ver_minor) <
-                (other.ver_major, other.ver_minor))
-
-    def __eq__(self, other):
-        if not isinstance(other, APIVersionRequest):
-            raise self._format_type_error(other)
-
-        return ((self.ver_major, self.ver_minor) ==
-                (other.ver_major, other.ver_minor))
-
-    def __gt__(self, other):
-        if not isinstance(other, APIVersionRequest):
-            raise self._format_type_error(other)
-
-        return ((self.ver_major, self.ver_minor) >
-                (other.ver_major, other.ver_minor))
-
-    def __le__(self, other):
-        return self < other or self == other
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __ge__(self, other):
-        return self > other or self == other
-
-    def matches(self, min_version, max_version):
-        """Returns whether the version object represents a version
-        greater than or equal to the minimum version and less than
-        or equal to the maximum version.
-
-        @param min_version: Minimum acceptable version.
-        @param max_version: Maximum acceptable version.
-        @returns: boolean
-
-        If min_version is null then there is no minimum limit.
-        If max_version is null then there is no maximum limit.
-        If self is null then raise ValueError
-        """
-
-        if self.is_null():
-            raise ValueError
-        if max_version.is_null() and min_version.is_null():
-            return True
-        elif max_version.is_null():
-            return min_version <= self
-        elif min_version.is_null():
-            return self <= max_version
-        else:
-            return min_version <= self <= max_version
-
-    def get_string(self):
-        """Converts object to string representation which if used to create
-        an APIVersionRequest object results in the same version request.
-        """
-        if self.is_null():
-            raise ValueError
-        return "%s.%s" % (self.ver_major, self.ver_minor)

cyborg/api/controllers/v2/device_profiles.py

@@ -29,10 +29,8 @@ from cyborg.api import expose
 from cyborg.common import exception
 from cyborg.common import policy
 from cyborg import objects
-
 LOG = log.getLogger(__name__)
 
-
 """
 The device profile object and db table has a profile_json field, which has
 its own version apart from the device profile groups field. The reasoning

cyborg/api/controllers/v2/versions.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2019 Intel, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+# This is the version 2 API
+BASE_VERSION = 2
+
+# String representations of the minor and maximum versions
+_MIN_VERSION_STRING = "2.0"
+_MAX_VERSION_STRING = "2.0"
+
+
+def service_type_string():
+    return 'accelerator'
+
+
+def min_version_string():
+    """Returns the minimum supported API version (as a string)"""
+    return _MIN_VERSION_STRING
+
+
+def max_version_string():
+    """Returns the maximum supported API version (as a string).
+
+    If the service is pinned, the maximum API version is the pinned
+    version. Otherwise, it is the maximum supported API version.
+
+    """
+    return _MAX_VERSION_STRING

cyborg/api/rest_api_version_history.rst

@@ -0,0 +1,22 @@
+REST API Version History
+========================
+
+This documents the changes made to the REST API with every
+microversion change. The description for each version should be a
+verbose one which has enough information to be suitable for use in
+user documentation.
+
+2.0
+---
+
+This is the initial version of the v2 API which supports
+microversions.
+
+A user can specify a header in the API request::
+
+  OpenStack-API-Version: accelerator <microversion>
+
+where ``<microversion>`` is any valid api microversion for this API.
+
+If no version is specified then the API will behave as if a version
+request of v2.0 was requested.

cyborg/tests/unit/api/base.py

@@ -145,7 +145,7 @@ class BaseApiTest(base.DbTestCase):
         return headers
 
     def get_json(self, path, expect_errors=False, headers=None,
-                 extra_environ=None, q=None, **params):
+                 extra_environ=None, q=None, return_json=True, **params):
         """Sends simulated HTTP GET request to Pecan test app.
 
         :param path: url path of target service
@@ -178,7 +178,7 @@ class BaseApiTest(base.DbTestCase):
                                 headers=headers,
                                 extra_environ=extra_environ,
                                 expect_errors=expect_errors)
-        if not expect_errors:
+        if return_json and not expect_errors:
             response = response.json
         return response
 

cyborg/tests/unit/api/controllers/v2/test_microversion.py

@@ -0,0 +1,96 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from cyborg.api.controllers.v2 import versions
+from cyborg.tests.unit.api import base as api_base
+
+
+SERVICE_TYPE = 'accelerator'
+H_MIN_VER = 'openstack-api-minimum-version'
+H_MAX_VER = 'openstack-api-maximum-version'
+H_RESP_VER = 'openstack-api-version'
+MIN_VER = versions.min_version_string()
+MAX_VER = versions.max_version_string()
+
+
+class TestMicroversions(api_base.BaseApiTest):
+
+    controller_list_response = [
+        'id', 'links', 'max_version', 'min_version', 'status']
+
+    def setUp(self):
+        super(TestMicroversions, self).setUp()
+
+    def test_wrong_major_version(self):
+        response = self.get_json(
+            '/v2',
+            headers={'OpenStack-API-Version': ' '.join([SERVICE_TYPE,
+                                                        '10'])},
+            expect_errors=True, return_json=False)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(406, response.status_int)
+        expected_error_msg = ('Invalid value for'
+                              ' OpenStack-API-Version header')
+        self.assertTrue(response.json['error_message'])
+        self.assertIn(expected_error_msg, response.json['error_message'])
+
+    def test_without_specified_microversion(self):
+        """If the header OpenStack-API-Version is absent in user's request,
+        the default microversion is MIN_VER.
+        """
+        response = self.get_json('/v2', return_json=False)
+        self.assertEqual(response.headers[H_MIN_VER], MIN_VER)
+        self.assertEqual(response.headers[H_MAX_VER], MAX_VER)
+        self.assertEqual(response.headers[H_RESP_VER], MIN_VER)
+        self.assertTrue(all(x in response.json.keys() for x in
+                            self.controller_list_response))
+
+    def test_new_client_new_api(self):
+        response = self.get_json(
+            '/v2',
+            headers={'OpenStack-API-Version': ' '.join([SERVICE_TYPE,
+                                                        '2.0'])},
+            return_json=False)
+        self.assertEqual(response.headers[H_MIN_VER], MIN_VER)
+        self.assertEqual(response.headers[H_MAX_VER], MAX_VER)
+        self.assertEqual(response.headers[H_RESP_VER], '2.0')
+        self.assertTrue(all(x in response.json.keys() for x in
+                            self.controller_list_response))
+
+    def test_latest_microversion(self):
+        response = self.get_json(
+            '/v2',
+            headers={'OpenStack-API-Version': ' '.join([SERVICE_TYPE,
+                                                        'latest'])},
+            return_json=False)
+        self.assertEqual(response.headers[H_MIN_VER], MIN_VER)
+        self.assertEqual(response.headers[H_MAX_VER], MAX_VER)
+        self.assertEqual(response.headers[H_RESP_VER], MAX_VER)
+        self.assertTrue(all(x in response.json.keys() for x in
+                            self.controller_list_response))
+
+    def test_unsupported_version(self):
+        unsupported_version = str(float(MAX_VER) + 0.1)
+        response = self.get_json(
+            '/v2',
+            headers={'OpenStack-API-Version': ' '.join(
+                [SERVICE_TYPE, unsupported_version])},
+            expect_errors=True)
+        self.assertEqual(406, response.status_int)
+        self.assertEqual(response.headers[H_MIN_VER], MIN_VER)
+        self.assertEqual(response.headers[H_MAX_VER], MAX_VER)
+        expected_error_msg = ('Version %s was requested but the minor '
+                              'version is not supported by this service. '
+                              'The supported version range is' %
+                              unsupported_version)
+        self.assertTrue(response.json['error_message'])
+        self.assertIn(expected_error_msg, response.json['error_message'])

releasenotes/notes/introduce-microversion-39c7f5cc6af4a139.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Cyborg now supports microversion in order to allow changes to the API while
+    preserving backward compatibility. User requests must include an HTTP header
+    ``OpenStack-API-Version: accelerator 2.0`` which is a monotonically increasing
+    semantic version number starting from 2.0. If that header is absent, the
+    request defaults to the default microverison 2.0.

requirements.txt

@@ -30,3 +30,4 @@ mock>=2.0.0 # BSD
 python-glanceclient>=2.3.0 # Apache-2.0
 oslo.privsep>=1.32.0 # Apache-2.0
 cursive>=0.2.1 # Apache-2.0
+microversion_parse>=0.2.1 # Apache-2.0