x/valence
Code Issues Proposed changes

Add device orchestration framework in valence

Browse Source 
This commit adds following functionalities:
- While creating podmanager all associated pooled resources
  will be synced.
- It provides user with following APIs:
  - List devices: v1/devices
  - Show device: v1/devices/<resourse_id>
  - Sync devices: v1/devices/sync

Change-Id: I5db45f5a7b4ffeec4b81758d8f719eaa4b5c9767
Partially-Implements: blueprint add-device-orchestration
This commit is contained in:
akhiljain23 2018-02-13 17:40:41 +05:30
parent 4eee1a4271
commit 0104b783ba
11 changed files with 482 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
valence/api/route.py
View File

@ -21,6 +21,7 @@ from six.moves import http_client
from valence.api import app as flaskapp
import valence.api.root as api_root
import valence.api.v1.devices as v1_devices
import valence.api.v1.flavors as v1_flavors
import valence.api.v1.nodes as v1_nodes
import valence.api.v1.podmanagers as v1_podmanagers
@ -108,5 +109,12 @@ api.add_resource(v1_podmanagers.PodManager,
api.add_resource(v1_podmanagers.PodManagersList,
'/v1/pod_managers', endpoint='podmanagers')
# Device(s) operations
api.add_resource(v1_devices.PooledDevicesList, '/v1/devices',
endpoint='devices')
api.add_resource(v1_devices.PooledDevices, '/v1/devices/<string:device_id>',
endpoint='device')
api.add_resource(v1_devices.SyncResources, '/v1/devices/sync', endpoint='sync')
# Proxy to PODM
api.add_resource(api_root.PODMProxy, '/<path:url>', endpoint='podmproxy')

52
valence/api/v1/devices.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright (c) 2017 NEC, Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
from flask import request
import flask_restful
from six.moves import http_client
from valence.common import utils
from valence.controller import pooled_devices
LOG = logging.getLogger(__name__)
class PooledDevicesList(flask_restful.Resource):
def get(self):
filters = request.args.to_dict()
return utils.make_response(
http_client.OK,
pooled_devices.PooledDevices.list_devices(filters))
class PooledDevices(flask_restful.Resource):
def get(self, device_id):
return utils.make_response(
http_client.OK,
pooled_devices.PooledDevices.get_device(device_id))
class SyncResources(flask_restful.Resource):
def post(self):
podm_id = None
if request.data:
podm_id = request.get_json().get('podm_id', None)
return utils.make_response(
http_client.OK,
pooled_devices.PooledDevices.synchronize_devices(podm_id))

7
valence/controller/podmanagers.py
View File

@ -17,6 +17,7 @@ import logging
from valence.common import exception
from valence.common import utils
from valence.controller import nodes
from valence.controller import pooled_devices
from valence.db import api as db_api
from valence.podmanagers import manager
@ -60,7 +61,11 @@ def create_podmanager(values):
# Retreive podm connection to get the status of podmanager
mng = manager.Manager(values['url'], username, password, values['driver'])
values['status'] = mng.podm.get_status()
return db_api.Connection.create_podmanager(values).as_dict()
podm = db_api.Connection.create_podmanager(values).as_dict()
# updates all devices corresponding to this podm in DB
# TODO(Akhil): Make this as asynchronous action
pooled_devices.PooledDevices.update_device_info(podm['uuid'])
return podm
def update_podmanager(uuid, values):

119
valence/controller/pooled_devices.py Normal file
View File

@ -0,0 +1,119 @@
# Copyright (c) 2017 NEC, Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
from valence.common import exception
from valence.db import api as db_api
from valence.podmanagers import manager
LOG = logging.getLogger(__name__)
class PooledDevices(object):
@staticmethod
def _show_device_brief_info(device_info):
return {key: device_info[key] for key in device_info.keys()
if key in ['uuid', 'podm_id', 'type', 'state', 'node_id',
'resource_uri', 'pooled_group_id']}
@classmethod
def list_devices(cls, filters={}):
"""List all registered devices
:param filters: filter by key, value arguments
Eg: {'podm_id': 'xxxx', 'type': 'SSD'}
:return: List of devices
"""
devices = db_api.Connection.list_devices(filters)
return [cls._show_device_brief_info(dev.as_dict()) for dev in devices]
@classmethod
def get_device(cls, device_id):
"""Get device info
:param device_id: UUID of device
:return: DB device info
"""
return db_api.Connection.get_device_by_uuid(device_id).as_dict()
@classmethod
def synchronize_devices(cls, podm_id=None):
"""Sync devices connected to podmanager(s)
It sync devices corresponding to particular podmanager
if podm_id is passed. Otherwise, all podmanagers will be
synced one by one.
:param podm_id: Optional podm_id to sync respective devices
:return: Podm_id and status message
"""
output = []
if podm_id:
LOG.debug('Synchronizing devices connected to podm %s', podm_id)
output.append(cls.update_device_info(podm_id))
return output
podms = db_api.Connection.list_podmanager()
for podm in podms:
LOG.debug('Synchronizing devices connected to podm %s',
podm['uuid'])
output.append(cls.update_device_info(podm['uuid']))
return output
@classmethod
def update_device_info(cls, podm_id):
"""Update/Add/Delete device info in DB
It compares all entries in database to data from connected
resources. Based on computation perform DB operation
(add/delete/update) on devices.
:param podm_id: UUID of podmanager
:return: Dictionary containing update status of podm
"""
LOG.debug('Update device info managed by podm %s started', podm_id)
response = dict()
response['podm_id'] = podm_id
try:
db_devices = db_api.Connection.list_devices({'podm_id': podm_id})
connection = manager.get_connection(podm_id)
podm_devices = {}
for device in connection.get_all_devices():
podm_devices[device['resource_uri']] = device
for db_dev in db_devices:
podm_dev = podm_devices.get(db_dev['resource_uri'], None)
if not podm_dev:
# device is disconnected, remove from db
db_api.Connection.delete_device(db_dev['uuid'])
continue
if db_dev['pooled_group_id'] != podm_dev['pooled_group_id']:
# update device info
values = {'pooled_group_id': podm_dev['pooled_group_id'],
'node_id': podm_dev['node_id'],
'state': podm_dev['state']
}
db_api.Connection.update_device(db_dev["uuid"], values)
del podm_devices[db_dev['resource_uri']]
continue
# remove device i.e already updated
del podm_devices[db_dev['resource_uri']]
# Add remaining devices available in podm_devices
for dev in podm_devices.values():
dev['podm_id'] = podm_id
db_api.Connection.add_device(dev)
response['status'] = 'SUCCESS'
except exception.ValenceException as e:
LOG.exception("Update devices failed with exception %s", str(e))
response['status'] = 'FAILED'
return response

4
valence/db/models.py
View File

@ -241,10 +241,10 @@ class Device(ModelBaseWithTimeStamp):
'validate': types.Text.validate
},
'properties': {
'validate': types.List(types.Dict).validate
'validate': types.Dict.validate
},
'extra': {
'validate': types.List(types.Dict).validate
'validate': types.Dict.validate
},
'resource_uri': {
'validate': types.Text.validate

4
valence/podmanagers/podm_base.py
View File

@ -58,6 +58,10 @@ class PodManagerBase(object):
def get_system_by_id(self, system_id):
pass
# TODO(): to be implemented in rsb_lib
def get_all_devices(self):
pass
def get_resource_info_by_url(self, resource_url):
return self.driver.get_resources_by_url(resource_url)

3
valence/tests/unit/api/test_route.py
View File

@ -49,4 +49,7 @@ class TestRoute(unittest.TestCase):
self.assertEqual(self.api.owns_endpoint('flavor'), True)
self.assertEqual(self.api.owns_endpoint('storages'), True)
self.assertEqual(self.api.owns_endpoint('storage'), True)
self.assertEqual(self.api.owns_endpoint('devices'), True)
self.assertEqual(self.api.owns_endpoint('device'), True)
self.assertEqual(self.api.owns_endpoint('sync'), True)
self.assertEqual(self.api.owns_endpoint('podmproxy'), True)

190
valence/tests/unit/controller/test_pooled_devices.py Normal file
View File

@ -0,0 +1,190 @@
# Copyright (c) 2017 NEC, Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import unittest
import mock
from valence.common import exception
from valence.controller import pooled_devices
from valence.podmanagers import podm_base
from valence.tests.unit.fakes import device_fakes as fakes
from valence.tests.unit.fakes import podmanager_fakes
class TestPooledDevices(unittest.TestCase):
@mock.patch('valence.db.api.Connection.list_devices')
def test_list_devices(self, mock_db_list_devices):
mock_db_list_devices.return_value = fakes.fake_device_obj_list()
result = pooled_devices.PooledDevices.list_devices()
self.assertEqual(fakes.fake_device_list(), result)
def test_show_device_brief_info(self):
device = fakes.fake_device()
expected = {
"node_id": None,
"podm_id": "88888888-8888-8888-8888-888888888888",
"pooled_group_id": "0000",
"resource_uri": "devices/0x7777777777",
"state": "free",
"type": "NIC",
"uuid": "00000000-0000-0000-0000-000000000000"
}
self.assertEqual(
expected,
pooled_devices.PooledDevices._show_device_brief_info(device))
@mock.patch('valence.controller.pooled_devices.PooledDevices.'
'update_device_info')
def test_synchronize_devices_with_podm_id(self, mock_update_device):
mock_update_device.return_value = {'podm_id': 'fake_uuid',
'status': 'SUCCESS'}
result = pooled_devices.PooledDevices.synchronize_devices('fake_uuid')
expected = [{'podm_id': 'fake_uuid', 'status': 'SUCCESS'}]
self.assertEqual(result, expected)
@mock.patch('valence.controller.pooled_devices.PooledDevices.'
'update_device_info')
@mock.patch('valence.db.api.Connection.list_podmanager')
def test_synchronize_devices_without_passing_podm_id(self,
mock_podm_list,
mock_update_device):
mock_podm_list.return_value = podmanager_fakes.fake_podmanager_list()
input = [{'podm_id': 'fake_uuid', 'status': 'FAILED'},
{'podm_id': 'fake_uuid', 'status': 'SUCCESS'}]
mock_update_device.side_effect = input
result = pooled_devices.PooledDevices.synchronize_devices()
self.assertEqual(result, input)
@mock.patch('valence.controller.pooled_devices.PooledDevices.'
'update_device_info')
def test_synchronize_devices_with_fail_device_update(self,
mock_update_device):
mock_update_device.return_value = {'podm_id': 'fake_uuid',
'status': 'FAILED'}
result = pooled_devices.PooledDevices.synchronize_devices('fake_uuid')
expected = [{'podm_id': 'fake_uuid', 'status': 'FAILED'}]
mock_update_device.assert_called_once_with('fake_uuid')
self.assertEqual(result, expected)
@mock.patch('valence.db.api.Connection.list_podmanager')
def test_synchronize_devices_no_podm_registered(self, mock_pod_db_list):
mock_pod_db_list.return_value = []
result = pooled_devices.PooledDevices.synchronize_devices()
self.assertEqual(result, [])
@mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
@mock.patch('valence.podmanagers.manager.get_connection')
@mock.patch('valence.db.api.Connection.list_devices')
@mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
def test_update_device_info_no_mismatch_device_info(self, mock_redfish,
mock_device_list,
mock_pod_conn,
mock_get_devices):
mock_device_list.return_value = fakes.fake_device_list()
mock_pod_conn.return_value = podm_base.PodManagerBase(
'fake', 'fake-pass', 'http://fake-url')
mock_get_devices.return_value = fakes.fake_device_list()
result = pooled_devices.PooledDevices.update_device_info('fake_id')
expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
self.assertEqual(result, expected)
@mock.patch('valence.db.api.Connection.update_device')
@mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
@mock.patch('valence.podmanagers.manager.get_connection')
@mock.patch('valence.db.api.Connection.list_devices')
@mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
def test_update_device_info_with_mismatch_device_info(self, mock_redfish,
mock_device_list,
mock_pod_conn,
mock_get_devices,
mock_update_device):
db_device_list = fakes.fake_device_list()
connected_devices = copy.deepcopy(db_device_list)
connected_devices[0]['pooled_group_id'] = '1234'
connected_devices[0]['node_id'] = '0x1234'
connected_devices[0]['state'] = 'free'
mock_device_list.return_value = db_device_list
mock_pod_conn.return_value = podm_base.PodManagerBase(
'fake', 'fake-pass', 'http://fake-url')
mock_get_devices.return_value = connected_devices
result = pooled_devices.PooledDevices.update_device_info('fake_id')
expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
values = {'pooled_group_id': '1234',
'node_id': '0x1234',
'state': 'free'
}
mock_update_device.assert_called_once_with(db_device_list[0]['uuid'],
values)
self.assertEqual(result, expected)
@mock.patch('valence.db.api.Connection.add_device')
@mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
@mock.patch('valence.podmanagers.manager.get_connection')
@mock.patch('valence.db.api.Connection.list_devices')
@mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
def test_update_device_info_with_new_device_connected(self, mock_redfish,
mock_device_list,
mock_pod_conn,
mock_get_devices,
mock_add_device):
device = fakes.fake_device()
connected_devices = [device]
mock_pod_conn.return_value = podm_base.PodManagerBase(
'fake', 'fake-pass', 'http://fake-url')
mock_get_devices.return_value = connected_devices
result = pooled_devices.PooledDevices.update_device_info('fake_id')
expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
device['podm_id'] = 'fake_id'
mock_add_device.assert_called_once_with(device)
self.assertEqual(result, expected)
@mock.patch('valence.db.api.Connection.delete_device')
@mock.patch('valence.db.api.Connection.add_device')
@mock.patch('valence.podmanagers.podm_base.PodManagerBase.get_all_devices')
@mock.patch('valence.podmanagers.manager.get_connection')
@mock.patch('valence.db.api.Connection.list_devices')
@mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
def test_update_device_info_with_disconnected_device(self, mock_redfish,
mock_device_list,
mock_pod_conn,
mock_get_devices,
mock_add_device,
mock_delete_device):
db_dev_list = fakes.fake_device_list()
mock_device_list.return_value = db_dev_list
device = fakes.fake_device()
connected_devices = [device]
mock_pod_conn.return_value = podm_base.PodManagerBase(
'fake', 'fake-pass', 'http://fake-url')
mock_get_devices.return_value = connected_devices
result = pooled_devices.PooledDevices.update_device_info('fake_id')
expected = {'podm_id': 'fake_id', 'status': 'SUCCESS'}
self.assertEqual(result, expected)
mock_delete_device.assert_any_call(db_dev_list[0]['uuid'])
mock_delete_device.assert_any_call(db_dev_list[1]['uuid'])
@mock.patch('valence.podmanagers.manager.get_connection')
@mock.patch('valence.db.api.Connection.list_devices')
@mock.patch('valence.redfish.sushy.sushy_instance.RedfishInstance')
def test_update_device_info_with_exception(self, mock_redfish,
mock_device_list,
mock_pod_conn):
mock_device_list.return_value = [fakes.fake_device()]
mock_pod_conn.side_effect = exception.ValenceException('fake_detail')
result = pooled_devices.PooledDevices.update_device_info('podm_id')
expected = {'podm_id': 'podm_id', 'status': 'FAILED'}
self.assertEqual(result, expected)

8
valence/tests/unit/db/utils.py
View File

@ -126,12 +126,8 @@ def get_test_device_db_info(**kwargs):
'pooled_group_id': kwargs.get('pooled_group_id', '2001'),
'state': kwargs.get('state', 'allocated'),
'properties': kwargs.get(
'properties',
[{'disk_size': '20'},
{'bandwidth': '100Mbps'}]),
'extra': kwargs.get(
'extra',
[{'mac': '11:11:11:11:11'}]),
'properties', {'disk_size': '20', 'bandwidth': '100Mbps'}),
'extra': kwargs.get('extra', {'mac': '11:11:11:11:11'}),
'resource_uri': kwargs.get('resource_uri', '/device/11'),
'created_at': kwargs.get('created_at', '2016-01-01 00:00:00 UTC'),
'updated_at': kwargs.get('updated_at', '2016-01-01 00:00:00 UTC')

58
valence/tests/unit/fakes/device_fakes.py Normal file
View File

@ -0,0 +1,58 @@
from valence.db import models
def fake_device():
return {
"created_at": "2018-01-18 10:36:29 UTC",
"extra": {
"device_name": "Qwerty device",
"vendor_name": "Qwerty Technologies"
},
"node_id": None,
"podm_id": "88888888-8888-8888-8888-888888888888",
"pooled_group_id": "0000",
"properties": {
"device_id": "0x7777777777",
"mac_address": "77:77:77:77:77:77"
},
"resource_uri": "devices/0x7777777777",
"state": "free",
"type": "NIC",
"updated_at": "2018-01-23 05:46:32 UTC",
"uuid": "00000000-0000-0000-0000-000000000000"
}
def fake_device_obj():
return models.Device(**fake_device())
def fake_device_list():
return [
{
"node_id": "0x11111111111",
"podm_id": "wwwwwwww-wwww-wwww-wwww-wwwwwwwwwwwwwwww",
"pooled_group_id": "1111",
"resource_uri": "devices/0x22222222222",
"state": "allocated",
"type": "NIC",
"uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
},
{
"node_id": None,
"podm_id": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy",
"pooled_group_id": "0000",
"resource_uri": "devices/0x666666666666",
"state": "free",
"type": "NIC",
"uuid": "zzzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzz"
}
]
def fake_device_obj_list():
values_list = fake_device_list()
for i in range(len(values_list)):
values_list[i] = models.Device(**values_list[i])
return values_list

38
valence/tests/unit/fakes/podmanager_fakes.py
View File

@ -32,3 +32,41 @@ def fake_podmanager():
def fake_podm_object():
return models.PodManager(**fake_podmanager())
def fake_podmanager_list():
return [
{
"authentication": [
{
"auth_items": {
"password": "***",
"username": "admin"
},
"type": "basic"
}],
"created_at": "2018-02-21 09:40:41 UTC",
"driver": "redfishv1",
"name": "podm1",
"status": "Online",
"updated_at": "2018-02-21 09:40:41 UTC",
"url": "http://127.0.0.1:0101",
"uuid": "0e7957c3-a28a-442d-b61c-0dd0dcb228d7"
},
{
"authentication": [
{
"auth_items": {
"password": "***",
"username": "admin"
},
"type": "basic"
}],
"created_at": "2018-02-21 09:40:41 UTC",
"driver": "redfishv1",
"name": "podm2",
"status": "Online",
"updated_at": "2018-02-21 09:40:41 UTC",
"url": "http://127.0.0.1:0000",
"uuid": "0e7957c3-a28a-442d-b61c-0dd0dcb228d6"
}]