Merge "Implementation of Device v2 API"

cyborg/api/controllers/types.py

@@ -30,10 +30,8 @@ class FilterType(wtypes.UserType):
     name = 'filtertype'
     basetype = wtypes.text
 
-    # TODO(Sundar): Ensure v1 and v2 APIs coexist.
     _supported_fields = wtypes.Enum(wtypes.text, 'parent_uuid', 'root_uuid',
-                                    'vendor', 'host', 'board', 'availability',
-                                    'assignable', 'interface_type',
+                                    'board', 'availability', 'interface_type',
                                     'instance_uuid', 'limit', 'marker',
                                     'sort_key', 'sort_dir')
 

cyborg/api/controllers/v2/__init__.py

@@ -24,6 +24,7 @@ from cyborg.api.controllers import link
 from cyborg.api.controllers.v2 import api_version_request
 from cyborg.api.controllers.v2 import arqs
 from cyborg.api.controllers.v2 import device_profiles
+from cyborg.api.controllers.v2 import devices
 from cyborg.api import expose
 
 
@@ -64,6 +65,7 @@ class Controller(rest.RestController):
 
     device_profiles = device_profiles.DeviceProfilesController()
     accelerator_requests = arqs.ARQsController()
+    devices = devices.DevicesController()
 
     @expose.expose(V2)
     def get(self):

cyborg/api/controllers/v2/devices.py

@@ -0,0 +1,130 @@
+# Copyright 2019 Intel, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+import wsme
+from wsme import types as wtypes
+
+from oslo_log import log
+
+from cyborg.api.controllers import base
+from cyborg.api.controllers import link
+from cyborg.api.controllers import types
+from cyborg.api import expose
+from cyborg.common import policy
+from cyborg import objects
+
+LOG = log.getLogger(__name__)
+
+
+class Device(base.APIBase):
+    """API representation of a device.
+
+    This class enforces type checking and value constraints, and converts
+    between the internal object model and the API representation.
+    """
+    uuid = types.uuid
+    """The UUID of the device"""
+
+    type = wtypes.text
+    """The type of the device"""
+
+    vendor = wtypes.text
+    """The vendor of the device"""
+
+    model = wtypes.text
+    """The model of the device"""
+
+    std_board_info = wtypes.text
+    """The standard board info of the device"""
+
+    vendor_board_info = wtypes.text
+    """The vendor board info of the device"""
+
+    hostname = wtypes.text
+    """The hostname of the device"""
+
+    links = wsme.wsattr([link.Link], readonly=True)
+    """A list containing a self link"""
+
+    def __init__(self, **kwargs):
+        super(Device, self).__init__(**kwargs)
+        self.fields = []
+        for field in objects.Device.fields:
+            self.fields.append(field)
+            setattr(self, field, kwargs.get(field, wtypes.Unset))
+
+    @classmethod
+    def convert_with_links(cls, obj_device):
+        api_device = cls(**obj_device.as_dict())
+        api_device.links = [
+            link.Link.make_link('self', pecan.request.public_url,
+                                'devices', api_device.uuid)
+            ]
+        return api_device
+
+
+class DeviceCollection(base.APIBase):
+    """API representation of a collection of devices."""
+
+    devices = [Device]
+    """A list containing Device objects"""
+
+    @classmethod
+    def convert_with_links(cls, devices):
+        collection = cls()
+        collection.devices = [Device.convert_with_links(device)
+                              for device in devices]
+        return collection
+
+
+class DevicesController(base.CyborgController):
+    """REST controller for Devices."""
+
+    @policy.authorize_wsgi("cyborg:device", "get_one")
+    @expose.expose(Device, wtypes.text)
+    def get_one(self, uuid):
+        """Get a single device by UUID.
+        :param uuid: uuid of a device.
+        """
+        context = pecan.request.context
+        device = objects.Device.get(context, uuid)
+        return Device.convert_with_links(device)
+
+    @policy.authorize_wsgi("cyborg:device", "get_all", False)
+    @expose.expose(DeviceCollection, wtypes.text, wtypes.text, wtypes.text,
+                   wtypes.ArrayType(types.FilterType))
+    def get_all(self, type=None, vendor=None, hostname=None, filters=None):
+        """Retrieve a list of devices.
+        :param type: type of a device.
+        :param vendor: vendor ID of a device.
+        :param hostname: the hostname of a compute node where the device
+        locates.
+        :param filters: a filter of FilterType to get device list by filter.
+        """
+        filters_dict = {}
+        if type:
+            filters_dict["type"] = type
+        if vendor:
+            filters_dict["vendor"] = vendor
+        if hostname:
+            filters_dict["hostname"] = hostname
+        if filters:
+            for filter in filters:
+                filters_dict.update(filter.as_dict())
+        context = pecan.request.context
+        obj_devices = objects.Device.list(context, filters=filters_dict)
+        LOG.info('[devices:get_all] Returned: %s', obj_devices)
+        return DeviceCollection.convert_with_links(obj_devices)

cyborg/common/policy.py

@@ -106,6 +106,15 @@ device_profile_policies = [
                        description='Delete device_profile records.'),
 ]
 
+device_policies = [
+    policy.RuleDefault('cyborg:device:get_one',
+                       'rule:allow',
+                       description='Show device detail'),
+    policy.RuleDefault('cyborg:device:get_all',
+                       'rule:allow',
+                       description='Retrieve all device records'),
+]
+
 fpga_policies = [
     policy.RuleDefault('cyborg:fpga:get_one',
                        'rule:allow',
@@ -123,7 +132,8 @@ def list_policies():
     return default_policies \
         + fpga_policies \
         + accelerator_request_policies \
-        + device_profile_policies
+        + device_profile_policies \
+        + device_policies
 
 
 @lockutils.synchronized('policy_enforcer', 'cyborg-')

cyborg/db/sqlalchemy/api.py

@@ -435,8 +435,8 @@ class Connection(api.Connection):
         return _paginate_query(context, models.Device, query_prefix,
                                limit, marker, sort_key, sort_dir)
 
-    def device_list(self, context, limit=None, marker=None,
-                    sort_key=None, sort_dir=None):
+    def device_list(self, context, limit=None, marker=None, sort_key=None,
+                    sort_dir=None):
         query = model_query(context, models.Device)
         return _paginate_query(context, models.Device, query,
                                limit, marker, sort_key, sort_dir)

cyborg/objects/device.py

@@ -66,11 +66,9 @@ class Device(base.CyborgObject, object_base.VersionedObjectDictCompat):
             sort_key = filters.pop('sort_key', 'created_at')
             limit = filters.pop('limit', None)
             marker = filters.pop('marker_obj', None)
-            db_devices = cls.dbapi.device_list_by_filters(context, filters,
-                                                          sort_dir=sort_dir,
-                                                          sort_key=sort_key,
-                                                          limit=limit,
-                                                          marker=marker)
+            db_devices = cls.dbapi.device_list_by_filters(
+                context, filters, sort_dir=sort_dir, sort_key=sort_key,
+                limit=limit, marker=marker)
         else:
             db_devices = cls.dbapi.device_list(context)
         return cls._from_db_object_list(db_devices, context)

cyborg/tests/unit/api/controllers/v2/test_devices.py

@@ -0,0 +1,120 @@
+# Copyright 2019 Intel, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from cyborg.tests.unit.api.controllers.v2 import base as v2_test
+from cyborg.tests.unit import fake_device
+
+
+class TestDevicesController(v2_test.APITestV2):
+
+    DEVICE_URL = '/devices'
+
+    def setUp(self):
+        super(TestDevicesController, self).setUp()
+        self.headers = self.gen_headers(self.context)
+        self.fake_devices = fake_device.get_fake_devices_objs()
+
+    def _validate_links(self, links, device_uuid):
+        has_self_link = False
+        for link in links:
+            if link['rel'] == 'self':
+                has_self_link = True
+                url = link['href']
+                components = url.split('/')
+                self.assertEqual(components[-1], device_uuid)
+        self.assertTrue(has_self_link)
+
+    def _validate_device(self, in_device, out_device):
+        for field in in_device.keys():
+            if field != 'id':
+                self.assertEqual(in_device[field], out_device[field])
+        # Check that the link is properly set up
+        self._validate_links(out_device['links'], in_device['uuid'])
+
+    @mock.patch('cyborg.objects.Device.get')
+    def test_get_one_by_uuid(self, mock_device):
+        in_device = self.fake_devices[0]
+        mock_device.return_value = in_device
+        uuid = in_device['uuid']
+
+        url = self.DEVICE_URL + '/%s'
+        out_device = self.get_json(url % uuid, headers=self.headers)
+        mock_device.assert_called_once()
+        self._validate_device(in_device, out_device)
+
+    @mock.patch('cyborg.objects.Device.list')
+    def test_get_all(self, mock_devices):
+        mock_devices.return_value = self.fake_devices
+        data = self.get_json(self.DEVICE_URL, headers=self.headers)
+        out_devices = data['devices']
+        self.assertIsInstance(out_devices, list)
+        for out_dev in out_devices:
+            self.assertIsInstance(out_dev, dict)
+        self.assertTrue(len(out_devices), len(self.fake_devices))
+        for in_device, out_device in zip(self.fake_devices, out_devices):
+            self._validate_device(in_device, out_device)
+
+    @mock.patch('cyborg.objects.Device.list')
+    def test_get_with_filters(self, mock_devices):
+        in_devices = self.fake_devices
+        mock_devices.return_value = in_devices[:1]
+        data = self.get_json(
+            self.DEVICE_URL + "?filters.field=limit&filters.value=1",
+            headers=self.headers)
+        out_devices = data['devices']
+        mock_devices.assert_called_once_with(mock.ANY, filters={"limit": "1"})
+        for in_device, out_device in zip(self.fake_devices, out_devices):
+            self._validate_device(in_device, out_device)
+
+    @mock.patch('cyborg.objects.Device.list')
+    def test_get_by_type(self, mock_devices):
+        in_devices = self.fake_devices
+        mock_devices.return_value = [in_devices[0]]
+        data = self.get_json(
+            self.DEVICE_URL + "?type=FPGA",
+            headers=self.headers)
+        out_devices = data['devices']
+        mock_devices.assert_called_once_with(mock.ANY,
+                                             filters={"type": "FPGA"})
+        for in_device, out_device in zip(self.fake_devices, out_devices):
+            self._validate_device(in_device, out_device)
+
+    @mock.patch('cyborg.objects.Device.list')
+    def test_get_by_vendor(self, mock_devices):
+        in_devices = self.fake_devices
+        mock_devices.return_value = [in_devices[0]]
+        data = self.get_json(
+            self.DEVICE_URL + "?vendor=0xABCD",
+            headers=self.headers)
+        out_devices = data['devices']
+        mock_devices.assert_called_once_with(mock.ANY,
+                                             filters={"vendor": "0xABCD"})
+        for in_device, out_device in zip(self.fake_devices, out_devices):
+            self._validate_device(in_device, out_device)
+
+    @mock.patch('cyborg.objects.Device.list')
+    def test_get_by_hostname(self, mock_devices):
+        in_devices = self.fake_devices
+        mock_devices.return_value = [in_devices[0]]
+        data = self.get_json(
+            self.DEVICE_URL + "?hostname=test-node-1",
+            headers=self.headers)
+        out_devices = data['devices']
+        mock_devices.assert_called_once_with(
+            mock.ANY, filters={"hostname": "test-node-1"})
+        for in_device, out_device in zip(self.fake_devices, out_devices):
+            self._validate_device(in_device, out_device)

cyborg/tests/unit/db/utils.py

@@ -110,25 +110,6 @@ def get_test_control_path(**kw):
     }
 
 
-def get_test_device(**kw):
-    return {
-        'id': kw.get('id', 1),
-        'uuid': kw.get('uuid', '83f92afa-1aa8-4548-a3a3-d218ff71d768'),
-        'type': kw.get('type', 'FPGA'),
-        'vendor': kw.get('vendor', 'Intel'),
-        # FIXME(Yumeng) should give details of std_board_info,
-        # vendor_board_info examples once they are determined.
-        'model': kw.get('model', 'PAC Arria 10'),
-        'std_board_info': kw.get('std_board_info',
-                                 'dictionary with standard fields'),
-        'vendor_board_info': kw.get('vendor_board_info',
-                                    'dictionary with vendor specific fields'),
-        'hostname': kw.get('hostname', 'hostname'),
-        'created_at': kw.get('create_at', None),
-        'updated_at': kw.get('updated_at', None),
-    }
-
-
 def get_test_device_profile(**kw):
     return {
         'id': kw.get('id', 1),

cyborg/tests/unit/fake_device.py

@@ -12,45 +12,62 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from oslo_utils import uuidutils
-
 from cyborg import objects
+from cyborg.objects import device
 from cyborg.objects import fields
 
 
-def fake_db_device(**updates):
-    root_uuid = uuidutils.generate_uuid()
-    db_device = {
-        'id': 1,
-        'uuid': root_uuid,
-        'type': 'FPGA',
-        'vendor': "vendor",
-        'model': "model",
-        'std_board_info': "std_board_info",
-        'vendor_board_info': "vendor_board_info",
-        'hostname': "hostname"
+def get_fake_devices_as_dict():
+    device1 = {
+        "id": 1,
+        "vendor": "0xABCD",
+        "uuid": u"1c6c9033-560d-4a7a-bb8e-94455d1e7825",
+        "hostname": "test-node-1",
+        "vendor_board_info": "fake_vendor_info",
+        "model": "miss model info",
+        "type": "FPGA",
+        "std_board_info": "{'class': 'Fake class', 'device_id': '0xabcd'}"
         }
+    device2 = {
+        "id": 2,
+        "vendor": "0xDCBA",
+        "uuid": u"1c6c9033-560d-4a7a-bb8e-94455d1e7826",
+        "hostname": "test-node-2",
+        "vendor_board_info": "fake_vendor_info",
+        "model": "miss model info",
+        "type": "GPU",
+        "std_board_info": "{'class': 'Fake class', 'device_id': '0xdcba'}"
+        }
+    return [device1, device2]
 
+
+def _convert_from_dict_to_obj(device_dict):
+    obj_device = device.Device()
+    for field in device_dict.keys():
+        obj_device[field] = device_dict[field]
+    return obj_device
+
+
+def _convert_to_db_device(device_dict):
     for name, field in objects.Device.fields.items():
-        if name in db_device:
+        if name in device_dict:
             continue
         if field.nullable:
-            db_device[name] = None
+            device_dict[name] = None
         elif field.default != fields.UnspecifiedDefault:
-            db_device[name] = field.default
+            device_dict[name] = field.default
         else:
             raise Exception('fake_db_device needs help with %s' % name)
-
-    if updates:
-        db_device.update(updates)
-
-    return db_device
+    return device_dict
 
 
-def fake_device_obj(context, obj_device_class=None, **updates):
-    if obj_device_class is None:
-        obj_device_class = objects.Device
-    device = obj_device_class._from_db_object(obj_device_class(),
-                                              fake_db_device(**updates))
-    device.obj_reset_changes()
-    return device
+def get_db_devices():
+    devices_list = get_fake_devices_as_dict()
+    db_devices = list(map(_convert_to_db_device, devices_list))
+    return db_devices
+
+
+def get_fake_devices_objs():
+    devices_list = get_fake_devices_as_dict()
+    obj_devices = list(map(_convert_from_dict_to_obj, devices_list))
+    return obj_devices

cyborg/tests/unit/objects/test_deployable.py

@@ -30,7 +30,7 @@ class TestDeployableObject(DbTestCase):
 
     @property
     def fake_device(self):
-        db_device = fake_device.fake_db_device(id=1)
+        db_device = fake_device.get_fake_devices_as_dict()[0]
         return db_device
 
     @property

cyborg/tests/unit/objects/test_device.py

@@ -17,14 +17,14 @@ import mock
 
 from cyborg import objects
 from cyborg.tests.unit.db import base
-from cyborg.tests.unit.db import utils
+from cyborg.tests.unit import fake_device
 
 
 class TestDeviceObject(base.DbTestCase):
 
     def setUp(self):
         super(TestDeviceObject, self).setUp()
-        self.fake_device = utils.get_test_device()
+        self.fake_device = fake_device.get_db_devices()[0]
 
     def test_get(self):
         uuid = self.fake_device['uuid']