Add device orchestration framework in valence

This commit adds following functionalities:
- While creating podmanager all associated pooled resources
  will be synced.
- It provides user with following APIs:
  - List devices: v1/devices
  - Show device: v1/devices/<resourse_id>
  - Sync devices: v1/devices/sync

Change-Id: I5db45f5a7b4ffeec4b81758d8f719eaa4b5c9767
Partially-Implements: blueprint add-device-orchestration

valence/api/route.py

@@ -21,6 +21,7 @@ from six.moves import http_client
 
 from valence.api import app as flaskapp
 import valence.api.root as api_root
+import valence.api.v1.devices as v1_devices
 import valence.api.v1.flavors as v1_flavors
 import valence.api.v1.nodes as v1_nodes
 import valence.api.v1.podmanagers as v1_podmanagers
@@ -108,5 +109,12 @@ api.add_resource(v1_podmanagers.PodManager,
 api.add_resource(v1_podmanagers.PodManagersList,
                  '/v1/pod_managers', endpoint='podmanagers')
 
+# Device(s) operations
+api.add_resource(v1_devices.PooledDevicesList, '/v1/devices',
+                 endpoint='devices')
+api.add_resource(v1_devices.PooledDevices, '/v1/devices/<string:device_id>',
+                 endpoint='device')
+api.add_resource(v1_devices.SyncResources, '/v1/devices/sync', endpoint='sync')
+
 # Proxy to PODM
 api.add_resource(api_root.PODMProxy, '/<path:url>', endpoint='podmproxy')

valence/api/v1/devices.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2017 NEC, Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+from flask import request
+import flask_restful
+from six.moves import http_client
+
+from valence.common import utils
+from valence.controller import pooled_devices
+
+LOG = logging.getLogger(__name__)
+
+
+class PooledDevicesList(flask_restful.Resource):
+
+    def get(self):
+        filters = request.args.to_dict()
+        return utils.make_response(
+            http_client.OK,
+            pooled_devices.PooledDevices.list_devices(filters))
+
+
+class PooledDevices(flask_restful.Resource):
+
+    def get(self, device_id):
+        return utils.make_response(
+            http_client.OK,
+            pooled_devices.PooledDevices.get_device(device_id))
+
+
+class SyncResources(flask_restful.Resource):
+
+    def post(self):
+        podm_id = None
+        if request.data:
+            podm_id = request.get_json().get('podm_id', None)
+        return utils.make_response(
+            http_client.OK,
+            pooled_devices.PooledDevices.synchronize_devices(podm_id))

valence/controller/podmanagers.py

@@ -17,6 +17,7 @@ import logging
 from valence.common import exception
 from valence.common import utils
 from valence.controller import nodes
+from valence.controller import pooled_devices
 from valence.db import api as db_api
 from valence.podmanagers import manager
 
@@ -60,7 +61,11 @@ def create_podmanager(values):
     # Retreive podm connection to get the status of podmanager
     mng = manager.Manager(values['url'], username, password, values['driver'])
     values['status'] = mng.podm.get_status()
-    return db_api.Connection.create_podmanager(values).as_dict()
+    podm = db_api.Connection.create_podmanager(values).as_dict()
+    # updates all devices corresponding to this podm in DB
+    # TODO(Akhil): Make this as asynchronous action
+    pooled_devices.PooledDevices.update_device_info(podm['uuid'])
+    return podm
 
 
 def update_podmanager(uuid, values):

valence/controller/pooled_devices.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2017 NEC, Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+from valence.common import exception
+from valence.db import api as db_api
+from valence.podmanagers import manager
+
+LOG = logging.getLogger(__name__)
+
+
+class PooledDevices(object):
+
+    @staticmethod
+    def _show_device_brief_info(device_info):
+        return {key: device_info[key] for key in device_info.keys()
+                if key in ['uuid', 'podm_id', 'type', 'state', 'node_id',
+                           'resource_uri', 'pooled_group_id']}
+
+    @classmethod
+    def list_devices(cls, filters={}):
+        """List all registered devices
+
+        :param filters: filter by key, value arguments
+        Eg: {'podm_id': 'xxxx', 'type': 'SSD'}
+        :return: List of devices
+        """
+        devices = db_api.Connection.list_devices(filters)
+        return [cls._show_device_brief_info(dev.as_dict()) for dev in devices]
+
+    @classmethod
+    def get_device(cls, device_id):
+        """Get device info
+
+        :param device_id: UUID of device
+        :return: DB device info
+        """
+        return db_api.Connection.get_device_by_uuid(device_id).as_dict()
+
+    @classmethod
+    def synchronize_devices(cls, podm_id=None):
+        """Sync devices connected to podmanager(s)
+
+        It sync devices corresponding to particular podmanager
+        if podm_id is passed. Otherwise, all podmanagers will be
+        synced one by one.
+        :param podm_id: Optional podm_id to sync respective devices
+        :return: Podm_id and status message
+        """
+        output = []
+        if podm_id:
+            LOG.debug('Synchronizing devices connected to podm %s', podm_id)
+            output.append(cls.update_device_info(podm_id))
+            return output
+        podms = db_api.Connection.list_podmanager()
+        for podm in podms:
+            LOG.debug('Synchronizing devices connected to podm %s',
+                      podm['uuid'])
+            output.append(cls.update_device_info(podm['uuid']))
+        return output
+
+    @classmethod
+    def update_device_info(cls, podm_id):
+        """Update/Add/Delete device info in DB
+
+        It compares all entries in database to data from connected
+        resources. Based on computation perform DB operation
+        (add/delete/update) on devices.
+        :param podm_id: UUID of podmanager
+        :return: Dictionary containing update status of podm
+        """
+        LOG.debug('Update device info managed by podm %s started', podm_id)
+        response = dict()
+        response['podm_id'] = podm_id
+        try:
+            db_devices = db_api.Connection.list_devices({'podm_id': podm_id})
+            connection = manager.get_connection(podm_id)
+            podm_devices = {}
+            for device in connection.get_all_devices():
+                podm_devices[device['resource_uri']] = device
+            for db_dev in db_devices:
+                podm_dev = podm_devices.get(db_dev['resource_uri'], None)
+                if not podm_dev:
+                    # device is disconnected, remove from db
+                    db_api.Connection.delete_device(db_dev['uuid'])
+                    continue
+                if db_dev['pooled_group_id'] != podm_dev['pooled_group_id']:
+                    # update device info
+                    values = {'pooled_group_id': podm_dev['pooled_group_id'],
+                              'node_id': podm_dev['node_id'],
+                              'state': podm_dev['state']
+                              }
+                    db_api.Connection.update_device(db_dev["uuid"], values)
+                    del podm_devices[db_dev['resource_uri']]
+                    continue
+                # remove device i.e already updated
+                del podm_devices[db_dev['resource_uri']]
+            # Add remaining devices available in podm_devices
+            for dev in podm_devices.values():
+                dev['podm_id'] = podm_id
+                db_api.Connection.add_device(dev)
+            response['status'] = 'SUCCESS'
+
+        except exception.ValenceException as e:
+            LOG.exception("Update devices failed with exception %s", str(e))
+            response['status'] = 'FAILED'
+        return response

valence/db/models.py

@@ -241,10 +241,10 @@ class Device(ModelBaseWithTimeStamp):
             'validate': types.Text.validate
         },
         'properties': {
-            'validate': types.List(types.Dict).validate
+            'validate': types.Dict.validate
         },
         'extra': {
-            'validate': types.List(types.Dict).validate
+            'validate': types.Dict.validate
         },
         'resource_uri': {
             'validate': types.Text.validate

valence/podmanagers/podm_base.py

@@ -58,6 +58,10 @@ class PodManagerBase(object):
     def get_system_by_id(self, system_id):
         pass
 
+    # TODO(): to be implemented in rsb_lib
+    def get_all_devices(self):
+        pass
+
     def get_resource_info_by_url(self, resource_url):
         return self.driver.get_resources_by_url(resource_url)
 

valence/tests/unit/api/test_route.py

@@ -49,4 +49,7 @@ class TestRoute(unittest.TestCase):
         self.assertEqual(self.api.owns_endpoint('flavor'), True)
         self.assertEqual(self.api.owns_endpoint('storages'), True)
         self.assertEqual(self.api.owns_endpoint('storage'), True)
+        self.assertEqual(self.api.owns_endpoint('devices'), True)
+        self.assertEqual(self.api.owns_endpoint('device'), True)
+        self.assertEqual(self.api.owns_endpoint('sync'), True)
         self.assertEqual(self.api.owns_endpoint('podmproxy'), True)

valence/tests/unit/controller/test_pooled_devices.py

@@ -0,0 +1,190 @@
+# Copyright (c) 2017 NEC, Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import unittest
+
+import mock
+
+from valence.common import exception
+from valence.controller import pooled_devices
+from valence.podmanagers import podm_base
+from valence.tests.unit.fakes import device_fakes as fakes
+from valence.tests.unit.fakes import podmanager_fakes
+
+
+class TestPooledDevices(unittest.TestCase):
+
+    @mock.patch('valence.db.api.Connection.list_devices')
+    def test_list_devices(self, mock_db_list_devices):
+        mock_db_list_devices.return_value = fakes.fake_device_obj_list()
+        result = pooled_devices.PooledDevices.list_devices()
+        self.assertEqual(fakes.fake_device_list(), result)
+
+    def test_show_device_brief_info(self):
+        device = fakes.fake_device()
+        expected = {
+            "node_id": None,
+            "podm_id": "88888888-8888-8888-8888-888888888888",
+            "pooled_group_id": "0000",
+            "resource_uri": "devices/0x7777777777",
+            "state": "free",
+            "type": "NIC",
+            "uuid": "00000000-0000-0000-0000-000000000000"
+        }
+        self.assertEqual(
+            expected,
+            pooled_devices.PooledDevices._show_device_brief_info(device))
+
+    @mock.patch('valence.controller.pooled_devices.PooledDevices.'
+                'update_device_info')
+    def test_synchronize_devices_with_podm_id(self, mock_update_device):
+        mock_update_device.return_value = {'podm_id': 'fake_uuid',
+                                           'status': 'SUCCESS'}
+        result = pooled_devices.PooledDevices.synchronize_devices('fake_uuid')
+        expected = [{'podm_id': 'fake_uuid', 'status': 'SUCCESS'}]
+        self.assertEqual(result, expected)
+
+    @mock.patch('valence.controller.pooled_devices.PooledDevices.'
+                'update_device_info')
+    @mock.patch('valence.db.api.Connection.list_podmanager')
+    def test_synchronize_devices_without_passing_podm_id(self,
+                                                         mock_podm_list,
+                                                         mock_update_device):
+        mock_podm_list.return_value = podmanager_fakes.fake_podmanager_list()
+        input = [{'podm_id': 'fake_uuid', 'status': 'FAILED'},
+                 {'podm_id': 'fake_uuid', 'status': 'SUCCESS'}]
+        mock_update_device.side_effect = input
+        result = pooled_devices.PooledDevices.synchronize_devices()
+        self.assertEqual(result, input)
+
+    @mock.patch('valence.controller.pooled_devices.PooledDevices.'
+                'update_device_info')
+    def test_synchronize_devices_with_fail_device_update(self,
+                                                         mock_update_device):
+        mock_update_device.return_value = {'podm_id': 'fake_uuid',
+                                           'status': 'FAILED'}
+        result = pooled_devices.PooledDevices.synchronize_devices('fake_uuid')
+        expected = [{'podm_id': 'fake_uuid', 'status': 'FAILED'}]
+        mock_update_device.assert_called_once_with('fake_uuid')
+        self.assertEqual(result, expected)
+
+    @mock.patch('valence.db.api.Connection.list_podmanager')
+    def test_synchronize_devices_no_podm_registered(self, mock_pod_db_list):
+        mock_pod_db_list.return_value = []
+        result = pooled_devices.PooledDevices.synchronize_devices()
+        self.assertEqual(result, [])
+
+    @mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
+    @mock.patch('valence.podmanagers.manager.get_connection')
+    @mock.patch('valence.db.api.Connection.list_devices')
+    @mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
+    def test_update_device_info_no_mismatch_device_info(self, mock_redfish,
+                                                        mock_device_list,
+                                                        mock_pod_conn,
+                                                        mock_get_devices):
+        mock_device_list.return_value = fakes.fake_device_list()
+        mock_pod_conn.return_value = podm_base.PodManagerBase(
+            'fake', 'fake-pass', 'http://fake-url')
+        mock_get_devices.return_value = fakes.fake_device_list()
+        result = pooled_devices.PooledDevices.update_device_info('fake_id')
+        expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
+        self.assertEqual(result, expected)
+
+    @mock.patch('valence.db.api.Connection.update_device')
+    @mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
+    @mock.patch('valence.podmanagers.manager.get_connection')
+    @mock.patch('valence.db.api.Connection.list_devices')
+    @mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
+    def test_update_device_info_with_mismatch_device_info(self, mock_redfish,
+                                                          mock_device_list,
+                                                          mock_pod_conn,
+                                                          mock_get_devices,
+                                                          mock_update_device):
+        db_device_list = fakes.fake_device_list()
+        connected_devices = copy.deepcopy(db_device_list)
+        connected_devices[0]['pooled_group_id'] = '1234'
+        connected_devices[0]['node_id'] = '0x1234'
+        connected_devices[0]['state'] = 'free'
+        mock_device_list.return_value = db_device_list
+        mock_pod_conn.return_value = podm_base.PodManagerBase(
+            'fake', 'fake-pass', 'http://fake-url')
+        mock_get_devices.return_value = connected_devices
+        result = pooled_devices.PooledDevices.update_device_info('fake_id')
+        expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
+        values = {'pooled_group_id': '1234',
+                  'node_id': '0x1234',
+                  'state': 'free'
+                  }
+        mock_update_device.assert_called_once_with(db_device_list[0]['uuid'],
+                                                   values)
+        self.assertEqual(result, expected)
+
+    @mock.patch('valence.db.api.Connection.add_device')
+    @mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
+    @mock.patch('valence.podmanagers.manager.get_connection')
+    @mock.patch('valence.db.api.Connection.list_devices')
+    @mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
+    def test_update_device_info_with_new_device_connected(self, mock_redfish,
+                                                          mock_device_list,
+                                                          mock_pod_conn,
+                                                          mock_get_devices,
+                                                          mock_add_device):
+        device = fakes.fake_device()
+        connected_devices = [device]
+        mock_pod_conn.return_value = podm_base.PodManagerBase(
+            'fake', 'fake-pass', 'http://fake-url')
+        mock_get_devices.return_value = connected_devices
+        result = pooled_devices.PooledDevices.update_device_info('fake_id')
+        expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
+        device['podm_id'] = 'fake_id'
+        mock_add_device.assert_called_once_with(device)
+        self.assertEqual(result, expected)
+
+    @mock.patch('valence.db.api.Connection.delete_device')
+    @mock.patch('valence.db.api.Connection.add_device')
+    @mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
+    @mock.patch('valence.podmanagers.manager.get_connection')
+    @mock.patch('valence.db.api.Connection.list_devices')
+    @mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
+    def test_update_device_info_with_disconnected_device(self, mock_redfish,
+                                                         mock_device_list,
+                                                         mock_pod_conn,
+                                                         mock_get_devices,
+                                                         mock_add_device,
+                                                         mock_delete_device):
+        db_dev_list = fakes.fake_device_list()
+        mock_device_list.return_value = db_dev_list
+        device = fakes.fake_device()
+        connected_devices = [device]
+        mock_pod_conn.return_value = podm_base.PodManagerBase(
+            'fake', 'fake-pass', 'http://fake-url')
+        mock_get_devices.return_value = connected_devices
+        result = pooled_devices.PooledDevices.update_device_info('fake_id')
+        expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
+        self.assertEqual(result, expected)
+        mock_delete_device.assert_any_call(db_dev_list[0]['uuid'])
+        mock_delete_device.assert_any_call(db_dev_list[1]['uuid'])
+
+    @mock.patch('valence.podmanagers.manager.get_connection')
+    @mock.patch('valence.db.api.Connection.list_devices')
+    @mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
+    def test_update_device_info_with_exception(self, mock_redfish,
+                                               mock_device_list,
+                                               mock_pod_conn):
+        mock_device_list.return_value = [fakes.fake_device()]
+        mock_pod_conn.side_effect = exception.ValenceException('fake_detail')
+        result = pooled_devices.PooledDevices.update_device_info('podm_id')
+        expected = {'podm_id': 'podm_id', 'status': 'FAILED'}
+        self.assertEqual(result, expected)

valence/tests/unit/db/utils.py

@@ -126,12 +126,8 @@ def get_test_device_db_info(**kwargs):
         'pooled_group_id': kwargs.get('pooled_group_id', '2001'),
         'state': kwargs.get('state', 'allocated'),
         'properties': kwargs.get(
-            'properties',
+            'properties', {'disk_size': '20', 'bandwidth': '100Mbps'}),
-            [{'disk_size': '20'},
+        'extra': kwargs.get('extra', {'mac': '11:11:11:11:11'}),
-             {'bandwidth': '100Mbps'}]),
-        'extra': kwargs.get(
-            'extra',
-            [{'mac': '11:11:11:11:11'}]),
         'resource_uri': kwargs.get('resource_uri', '/device/11'),
         'created_at': kwargs.get('created_at', '2016-01-01 00:00:00 UTC'),
         'updated_at': kwargs.get('updated_at', '2016-01-01 00:00:00 UTC')

valence/tests/unit/fakes/device_fakes.py

@@ -0,0 +1,58 @@
+from valence.db import models
+
+
+def fake_device():
+    return {
+        "created_at": "2018-01-18 10:36:29 UTC",
+        "extra": {
+            "device_name": "Qwerty device",
+            "vendor_name": "Qwerty Technologies"
+        },
+        "node_id": None,
+        "podm_id": "88888888-8888-8888-8888-888888888888",
+        "pooled_group_id": "0000",
+        "properties": {
+            "device_id": "0x7777777777",
+            "mac_address": "77:77:77:77:77:77"
+        },
+        "resource_uri": "devices/0x7777777777",
+        "state": "free",
+        "type": "NIC",
+        "updated_at": "2018-01-23 05:46:32 UTC",
+        "uuid": "00000000-0000-0000-0000-000000000000"
+    }
+
+
+def fake_device_obj():
+    return models.Device(**fake_device())
+
+
+def fake_device_list():
+    return [
+        {
+            "node_id": "0x11111111111",
+            "podm_id": "wwwwwwww-wwww-wwww-wwww-wwwwwwwwwwwwwwww",
+            "pooled_group_id": "1111",
+            "resource_uri": "devices/0x22222222222",
+            "state": "allocated",
+            "type": "NIC",
+            "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+        },
+        {
+            "node_id": None,
+            "podm_id": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy",
+            "pooled_group_id": "0000",
+            "resource_uri": "devices/0x666666666666",
+            "state": "free",
+            "type": "NIC",
+            "uuid": "zzzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzz"
+        }
+        ]
+
+
+def fake_device_obj_list():
+    values_list = fake_device_list()
+    for i in range(len(values_list)):
+        values_list[i] = models.Device(**values_list[i])
+
+    return values_list

valence/tests/unit/fakes/podmanager_fakes.py

@@ -32,3 +32,41 @@ def fake_podmanager():
 
 def fake_podm_object():
     return models.PodManager(**fake_podmanager())
+
+
+def fake_podmanager_list():
+    return [
+        {
+            "authentication": [
+                {
+                    "auth_items": {
+                        "password": "***",
+                        "username": "admin"
+                    },
+                    "type": "basic"
+                }],
+            "created_at": "2018-02-21 09:40:41 UTC",
+            "driver": "redfishv1",
+            "name": "podm1",
+            "status": "Online",
+            "updated_at": "2018-02-21 09:40:41 UTC",
+            "url": "http://127.0.0.1:0101",
+            "uuid": "0e7957c3-a28a-442d-b61c-0dd0dcb228d7"
+        },
+        {
+            "authentication": [
+                {
+                    "auth_items": {
+                        "password": "***",
+                        "username": "admin"
+                    },
+                    "type": "basic"
+                }],
+            "created_at": "2018-02-21 09:40:41 UTC",
+            "driver": "redfishv1",
+            "name": "podm2",
+            "status": "Online",
+            "updated_at": "2018-02-21 09:40:41 UTC",
+            "url": "http://127.0.0.1:0000",
+            "uuid": "0e7957c3-a28a-442d-b61c-0dd0dcb228d6"
+        }]