starlingx
/
config
Code Issues Proposed changes

Merge "Update device image state in device label API"

This commit is contained in:
Zuul 2020-06-05 16:38:54 +00:00 committed by Gerrit Code Review
parent a1576972fa da2cb70699
commit 2b40b2a711
12 changed files with 261 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sysinv/cgts-client/cgts-client/cgtsclient/v1/pci_device_shell.py
View File

@ -26,11 +26,11 @@ def _print_device_show(device):
pclass_id = getattr(device, 'pclass_id')
if pclass_id == PCI_DEVICE_CLASS_FPGA:
fields += ['needs_firmware_update', 'status', 'root_key',
'revoked_key_ids', 'boot_page', 'bitstream_id',
fields += ['root_key', 'revoked_key_ids',
'boot_page', 'bitstream_id',
'bmc_build_version', 'bmc_fw_version']
labels += ['needs_firmware_update', 'status', 'root_key',
'revoked_key_ids', 'boot_page', 'bitstream_id',
labels += ['root_key', 'revoked_key_ids',
'boot_page', 'bitstream_id',
'bmc_build_version', 'bmc_fw_version']
data = [(f, getattr(device, f, '')) for f in fields]

19
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/device_image.py
View File

@ -158,7 +158,10 @@ def _get_applied_labels(device_image):
applied_labels = {}
for image_label in image_labels:
label = pecan.request.dbapi.device_label_get(image_label.label_uuid)
applied_labels[label.label_key] = label.label_value
applied_labels.setdefault(label.label_key, [])
if label.label_value not in applied_labels[label.label_key]:
applied_labels[label.label_key].append(label.label_value)
device_image.applied_labels = applied_labels
return device_image
@ -259,10 +262,16 @@ class DeviceImageController(rest.RestController):
@wsme_pecan.wsexpose(None, types.uuid, status_code=204)
def delete(self, deviceimage_uuid):
"""Delete a device image."""
# TODO Only allow delete if there are no devices using the image
device_image = objects.device_image.get_by_uuid(
pecan.request.context, deviceimage_uuid)
# Check if the image has been written to any of the devices
if pecan.request.dbapi.device_image_state_get_all(
image_id=device_image.id,
status=dconstants.DEVICE_IMAGE_UPDATE_COMPLETED):
raise wsme.exc.ClientSideError(_(
"Delete failed: device image has already been written to devices"))
filename = cutils.format_image_filename(device_image)
pecan.request.rpcapi.delete_bitstream_file(pecan.request.context,
filename)
@ -466,10 +475,6 @@ def delete_device_image_state(pcidevice_id, device_image):
def modify_flags(pcidevice_id, host_id):
# Set flag for pci_device indicating device requires image update
pecan.request.dbapi.pci_device_update(pcidevice_id,
{'needs_firmware_update': True},
host_id)
# Set flag for host indicating device image update is pending if it is
# not already in progress
host = pecan.request.dbapi.ihost_get(host_id)

68
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/device_label.py
View File

@ -17,6 +17,7 @@ from sysinv.api.controllers.v1 import collection
from sysinv.api.controllers.v1 import types
from sysinv.api.controllers.v1 import utils
from sysinv.common import exception
from sysinv.common import device as dconstants
from sysinv.common import utils as cutils
from sysinv import objects
@ -182,44 +183,42 @@ class DeviceLabelController(rest.RestController):
del body['pcidevice_uuid']
pcidevice = objects.pci_device.get_by_uuid(pecan.request.context,
pcidevice_uuid)
fpgadevice = pecan.request.dbapi.fpga_device_get(pcidevice.pciaddr,
pcidevice.host_id)
existing_labels = {}
for label_key in body.keys():
label = None
try:
label = pecan.request.dbapi.device_label_query(
pcidevice.id, label_key)
except exception.DeviceLabelNotFoundByKey:
pass
if label:
if overwrite:
existing_labels.update({label_key: label.uuid})
labels = pecan.request.dbapi.device_label_query(
pcidevice.id, label_key)
if len(labels) == 0:
continue
if overwrite:
if len(labels) == 1:
existing_labels.update({label_key: labels[0].uuid})
else:
raise wsme.exc.ClientSideError(_(
"Label %s exists for device %s. Use overwrite option"
" to assign a new value." %
(label_key, pcidevice.name)))
"Cannot overwrite label value as multiple device "
"labels exist with label key %s for this device") % label_key)
new_records = []
for key, value in body.items():
values = {
'host_id': pcidevice.host_id,
'pcidevice_id': pcidevice.id,
'fpgadevice_id': fpgadevice.id,
'label_key': key,
'label_value': value
}
try:
if existing_labels.get(key, None):
# Update the value
# Label exists, need to remove the existing
# device_image_state entries for the old label and
# create new entries for the updated label if needed
label_uuid = existing_labels.get(key)
remove_device_image_state(label_uuid)
new_label = pecan.request.dbapi.device_label_update(
label_uuid, {'label_value': value})
else:
new_label = pecan.request.dbapi.device_label_create(
pcidevice_uuid, values)
add_device_image_state(pcidevice, key, value)
new_records.append(new_label)
except exception.DeviceLabelAlreadyExists:
# We should not be here
@ -234,7 +233,7 @@ class DeviceLabelController(rest.RestController):
@wsme_pecan.wsexpose(None, types.uuid, status_code=204)
def delete(self, device_label_uuid):
"""Delete a device label."""
remove_device_image_state(device_label_uuid)
pecan.request.dbapi.device_label_destroy(device_label_uuid)
@cutils.synchronized(LOCK_NAME)
@ -242,3 +241,38 @@ class DeviceLabelController(rest.RestController):
def patch(self, device_label):
"""Modify a new device label."""
raise exception.OperationNotPermitted
def remove_device_image_state(device_label_uuid):
"""Cleanup device image state based on device label"""
device_label = pecan.request.dbapi.device_label_get(device_label_uuid)
host = pecan.request.dbapi.ihost_get(device_label.host_uuid)
if host.device_image_update == dconstants.DEVICE_IMAGE_UPDATE_IN_PROGRESS:
raise wsme.exc.ClientSideError(_(
"Command rejected: Device image update is in progress for host %s" %
host.hostname))
dev_img_states = pecan.request.dbapi.device_image_state_get_all(
host_id=device_label.host_id,
pcidevice_id=device_label.pcidevice_id)
for state in dev_img_states:
pecan.request.dbapi.device_image_state_destroy(state.id)
def add_device_image_state(pcidevice, key, value):
"""Add device image state based on device label"""
# If the image has been applied to any devices,
# create device_image_state entry for this device
dev_labels = pecan.request.dbapi.device_label_get_by_label(key, value)
if dev_labels:
dev_img_lbls = pecan.request.dbapi.device_image_label_get_by_label(
dev_labels[0].id)
for img in dev_img_lbls:
# Create an entry of image to device mapping
state_values = {
'host_id': pcidevice.host_id,
'pcidevice_id': pcidevice.id,
'image_id': img.image_id,
'status': dconstants.DEVICE_IMAGE_UPDATE_PENDING,
}
pecan.request.dbapi.device_image_state_create(state_values)

7
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/pci_device.py
View File

@ -122,12 +122,6 @@ class PCIDevice(base.APIBase):
bitstream_id = wtypes.text
"Represent the bitstream id of the fpga device"
needs_firmware_update = types.boolean
"Represent whether firmware update is required for the fpga device"
status = wtypes.text
"Represent the status of the fpga device"
links = [link.Link]
"Represent a list containing a self link and associated device links"
@ -151,7 +145,6 @@ class PCIDevice(base.APIBase):
'bmc_build_version', 'bmc_fw_version',
'root_key', 'revoked_key_ids',
'boot_page', 'bitstream_id',
'needs_firmware_update', 'status',
'created_at', 'updated_at'])
# do not expose the id attribute

14
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py
View File

@ -8592,11 +8592,7 @@ class Connection(api.Connection):
query = model_query(models.DeviceLabel, session=session)
query = query.filter(models.DeviceLabel.pcidevice_id == device_id)
query = query.filter(models.DeviceLabel.label_key == label_key)
try:
result = query.one()
except NoResultFound:
raise exception.DeviceLabelNotFoundByKey(label=label_key)
return result
return query.all()
@objects.objectify(objects.device_label)
def device_label_query(self, device_id, label_key):
@ -8663,6 +8659,14 @@ class Connection(api.Connection):
query = query.filter_by(image_id=image_id)
return query.all()
@objects.objectify(objects.device_image_label)
def device_image_label_get_by_label(self, label_id,
limit=None, marker=None,
sort_key=None, sort_dir=None):
query = model_query(models.DeviceImageLabel)
query = query.filter_by(label_id=label_id)
return query.all()
@objects.objectify(objects.device_image_label)
def device_image_label_get_by_image_label(self, image_id, label_id,
limit=None, marker=None,

39
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/106_fpga_remove_columns.py Normal file
View File

@ -0,0 +1,39 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from sqlalchemy import Column, MetaData, Table
from migrate.changeset import UniqueConstraint
ENGINE = 'InnoDB'
CHARSET = 'utf8'
def upgrade(migrate_engine):
"""
This database upgrade removes unused attributes
from pci_devices and device_labels tables.
"""
meta = MetaData()
meta.bind = migrate_engine
pci_devices = Table('pci_devices', meta, autoload=True)
pci_devices.drop_column(Column('status'))
pci_devices.drop_column(Column('needs_firmware_update'))
device_labels = Table('device_labels', meta, autoload=True)
device_labels.drop_column(Column('fpgadevice_id'))
UniqueConstraint('pcidevice_id', 'label_key', table=device_labels,
name='u_pcidevice_id@label_key').drop()
return True
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
# Downgrade is unsupported.
raise NotImplementedError('SysInv database downgrade is unsupported.')

7
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py
View File

@ -1462,9 +1462,6 @@ class PciDevice(Base):
enabled = Column(Boolean)
extra_info = Column(Text)
status = Column(String(128))
needs_firmware_update = Column(Boolean, nullable=False, default=False)
host = relationship("ihost", lazy="joined", join_depth=1)
fpga = relationship("FpgaDevice", lazy="joined", uselist=False, join_depth=1)
UniqueConstraint('pciaddr', 'host_id', name='u_pciaddrhost')
@ -1572,16 +1569,12 @@ class DeviceLabel(Base):
host_id = Column(Integer, ForeignKey('i_host.id', ondelete='CASCADE'))
pcidevice_id = Column(Integer, ForeignKey('pci_devices.id',
ondelete='CASCADE'))
fpgadevice_id = Column(Integer, ForeignKey('fpga_devices.id',
ondelete='CASCADE'))
capabilities = Column(JSONEncodedDict)
host = relationship("ihost", lazy="joined", join_depth=1)
pcidevice = relationship("PciDevice", lazy="joined", join_depth=1)
fpgadevice = relationship("FpgaDevice", lazy="joined", join_depth=1)
label_key = Column(String(384))
label_value = Column(String(128))
UniqueConstraint('pcidevice_id', 'label_key', name='u_pcidevice_id@label_key')
class DeviceImageLabel(Base):

2
sysinv/sysinv/sysinv/sysinv/objects/device_label.py
View File

@ -23,8 +23,6 @@ class DeviceLabel(base.SysinvObject):
'label_value': utils.str_or_none,
'pcidevice_id': utils.int_or_none,
'pcidevice_uuid': utils.str_or_none,
'fpgadevice_id': utils.int_or_none,
'fpgadevice_uuid': utils.str_or_none,
'capabilities': utils.dict_or_none,
}

2
sysinv/sysinv/sysinv/sysinv/objects/pci_device.py
View File

@ -42,8 +42,6 @@ class PCIDevice(base.SysinvObject):
'revoked_key_ids': utils.str_or_none,
'boot_page': utils.str_or_none,
'bitstream_id': utils.str_or_none,
'status': utils.str_or_none,
'needs_firmware_update': utils.bool_or_none,
}
_foreign_fields = {

14
sysinv/sysinv/sysinv/sysinv/tests/api/test_device_image.py
View File

@ -329,10 +329,6 @@ class TestPatch(TestDeviceImage):
self.assertEqual(dconstants.DEVICE_IMAGE_UPDATE_PENDING,
dev_img_state.status)
# Verify that needs_firmware_update flag is updated in pci_device
pci_dev = self.dbapi.pci_device_get(self.pci_device.id)
self.assertEqual(pci_dev['needs_firmware_update'], True)
def test_device_image_apply_invalid_image(self):
# Test applying device image with non-existing image
@ -366,7 +362,7 @@ class TestPatch(TestDeviceImage):
self.assertEqual(response.json['pci_vendor'], '80ee')
self.assertEqual(response.json['pci_device'], 'beef')
self.assertEqual(response.json['bitstream_id'], '12345')
self.assertEqual(response.json['applied_labels'], {'key1': 'value1'})
self.assertEqual(response.json['applied_labels'], {'key1': ['value1']})
# Verify that the image to device mapping is updated
dev_img_state = self.dbapi.device_image_state_get_by_image_device(
@ -374,10 +370,6 @@ class TestPatch(TestDeviceImage):
self.assertEqual(dconstants.DEVICE_IMAGE_UPDATE_PENDING,
dev_img_state.status)
# Verify that needs_firmware_update flag is updated in pci_device
pci_dev = self.dbapi.pci_device_get(self.pci_device.id)
self.assertEqual(pci_dev['needs_firmware_update'], True)
def test_device_image_apply_invalid_label(self):
# Test applying device image with non-existing device label
@ -429,10 +421,6 @@ class TestPatch(TestDeviceImage):
self.assertEqual(response.json['pci_device'], 'beef')
self.assertEqual(response.json['bitstream_id'], '12345')
# Verify that needs_firmware_update flag is updated in pci_device
pci_dev = self.dbapi.pci_device_get(self.pci_device.id)
self.assertEqual(pci_dev['needs_firmware_update'], False)
def test_device_image_remove_by_label(self):
# Test removing device image by device label

288
sysinv/sysinv/sysinv/sysinv/tests/api/test_device_label.py
View File

@ -1,143 +1,145 @@
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import json
from six.moves import http_client
from six.moves.urllib.parse import urlencode
from sysinv.db import api as dbapi
from sysinv.tests.api import base
from sysinv.tests.db import base as dbbase
from sysinv.tests.db import utils as dbutils
class DeviceLabelTestCase(base.FunctionalTest, dbbase.ControllerHostTestCase):
def setUp(self):
super(DeviceLabelTestCase, self).setUp()
self.dbapi = dbapi.get_instance()
# Create a pci_device and fpga_device object
self.pci_device = dbutils.create_test_pci_devices(
host_id=self.host.id,
pclass='Processing accelerators',
pclass_id='120000',)
self.fpga_device = dbutils.create_test_fpga_device(
host_id=self.host.id,
pci_id=self.pci_device.id)
self.generic_labels = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value1',
'key2': 'value2'
}
def _get_path(self, params=None):
path = '/device_labels'
if params:
path += '?' + urlencode(params)
return path
def validate_labels(self, input_data, response_data):
for t in response_data:
for k, v in t.items():
if k in input_data.keys():
self.assertEqual(v, input_data[k])
def assign_labels(self, input_data, parameters=None):
response = self.post_json('%s' % self._get_path(parameters), input_data)
self.assertEqual(http_client.OK, response.status_int)
return response
def assign_labels_failure(self, input_data, parameters=None):
response = self.post_json('%s' % self._get_path(parameters), input_data,
expect_errors=True)
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
def get_device_labels(self):
response = self.get_json("/device_labels")
return response['device_labels']
class DeviceLabelAssignTestCase(DeviceLabelTestCase):
def setUp(self):
super(DeviceLabelAssignTestCase, self).setUp()
def test_create_device_labels(self):
self.assign_labels(self.generic_labels)
response_data = self.get_device_labels()
self.validate_labels(self.generic_labels, response_data)
def test_overwrite_device_labels_success(self):
self.assign_labels(self.generic_labels)
new_input_values = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'string1',
'key2': 'string2'
}
self.assign_labels(new_input_values, parameters={'overwrite': True})
response_data = self.get_device_labels()
self.validate_labels(new_input_values, response_data)
def test_overwrite_device_labels_failure(self):
self.assign_labels(self.generic_labels)
new_input_values = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'string1',
'key2': 'string2'
}
# Default value should be overwrite=False
self.assign_labels_failure(new_input_values)
# Test explicit overwrite=False
self.assign_labels_failure(new_input_values, parameters={'overwrite': False})
# Labels should be unchanged from initial values
response_data = self.get_device_labels()
self.validate_labels(self.generic_labels, response_data)
def test_create_validated_device_labels_success(self):
label1 = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value1',
}
self.assign_labels(label1)
label2 = {
'pcidevice_uuid': self.pci_device.uuid,
'key2': 'value2',
}
self.assign_labels(label2)
input_data = {}
for input_label in [label1, label2]:
input_data.update(input_label)
response_data = self.get_device_labels()
self.validate_labels(input_data, response_data)
class DeviceLabelRemoveTestCase(DeviceLabelTestCase):
def setUp(self):
super(DeviceLabelRemoveTestCase, self).setUp()
def test_remove_device_labels(self):
# Assign labels to a device
response = self.assign_labels(self.generic_labels)
resp = json.loads(response.body)
self.assertIn('device_labels', resp)
resp_dict = resp.get('device_labels')
uuid = resp_dict[0]['uuid']
# Remove a label from the device
self.delete('/device_labels/%s' % uuid,
headers={'User-Agent': 'sysinv-test'})
# Verify the device label no longer exists
response = self.get_json('/device_labels/%s' % uuid,
expect_errors=True)
self.assertEqual(response.status_int, http_client.BAD_REQUEST)
self.assertEqual(response.content_type, 'application/json')
self.assertTrue(response.json['error_message'])
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import json
from six.moves import http_client
from six.moves.urllib.parse import urlencode
from sysinv.db import api as dbapi
from sysinv.tests.api import base
from sysinv.tests.db import base as dbbase
from sysinv.tests.db import utils as dbutils
class DeviceLabelTestCase(base.FunctionalTest, dbbase.ControllerHostTestCase):
def setUp(self):
super(DeviceLabelTestCase, self).setUp()
self.dbapi = dbapi.get_instance()
# Create a pci_device and fpga_device object
self.pci_device = dbutils.create_test_pci_devices(
host_id=self.host.id,
pclass='Processing accelerators',
pclass_id='120000',)
self.fpga_device = dbutils.create_test_fpga_device(
host_id=self.host.id,
pci_id=self.pci_device.id)
self.generic_labels = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value1',
'key2': 'value2'
}
def _get_path(self, params=None):
path = '/device_labels'
if params:
path += '?' + urlencode(params)
return path
def validate_labels(self, input_data, response_data):
for t in response_data:
for k, v in t.items():
if k in input_data.keys():
self.assertEqual(v, input_data[k])
def assign_labels(self, input_data, parameters=None):
response = self.post_json('%s' % self._get_path(parameters), input_data)
self.assertEqual(http_client.OK, response.status_int)
return response
def assign_labels_failure(self, input_data, parameters=None):
response = self.post_json('%s' % self._get_path(parameters), input_data,
expect_errors=True)
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
def get_device_labels(self):
response = self.get_json("/device_labels")
return response['device_labels']
class DeviceLabelAssignTestCase(DeviceLabelTestCase):
def setUp(self):
super(DeviceLabelAssignTestCase, self).setUp()
def test_create_device_labels(self):
self.assign_labels(self.generic_labels)
response_data = self.get_device_labels()
self.validate_labels(self.generic_labels, response_data)
def test_overwrite_device_labels_success(self):
self.assign_labels(self.generic_labels)
new_input_values = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'string1',
'key2': 'string2'
}
self.assign_labels(new_input_values, parameters={'overwrite': True})
response_data = self.get_device_labels()
self.validate_labels(new_input_values, response_data)
def test_overwrite_device_labels_failure(self):
label1 = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value1',
}
self.assign_labels(label1)
label2 = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value2',
}
self.assign_labels(label2)
new_input_values = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'string1',
}
self.assign_labels_failure(new_input_values, parameters={'overwrite': True})
def test_create_validated_device_labels_success(self):
label1 = {
'pcidevice_uuid': self.pci_device.uuid,
'key1': 'value1',
}
self.assign_labels(label1)
label2 = {
'pcidevice_uuid': self.pci_device.uuid,
'key2': 'value2',
}
self.assign_labels(label2)
input_data = {}
for input_label in [label1, label2]:
input_data.update(input_label)
response_data = self.get_device_labels()
self.validate_labels(input_data, response_data)
class DeviceLabelRemoveTestCase(DeviceLabelTestCase):
def setUp(self):
super(DeviceLabelRemoveTestCase, self).setUp()
def test_remove_device_labels(self):
# Assign labels to a device
response = self.assign_labels(self.generic_labels)
resp = json.loads(response.body)
self.assertIn('device_labels', resp)
resp_dict = resp.get('device_labels')
uuid = resp_dict[0]['uuid']
# Remove a label from the device
self.delete('/device_labels/%s' % uuid,
headers={'User-Agent': 'sysinv-test'})
# Verify the device label no longer exists
response = self.get_json('/device_labels/%s' % uuid,
expect_errors=True)
self.assertEqual(response.status_int, http_client.BAD_REQUEST)
self.assertEqual(response.content_type, 'application/json')
self.assertTrue(response.json['error_message'])

2
sysinv/sysinv/sysinv/sysinv/tests/db/utils.py
View File

@ -1339,8 +1339,6 @@ def get_test_fpga_device(**kw):
'revoked_key_ids': kw.get('revoked_key_ids'),
'boot_page': kw.get('boot_page'),
'bitstream_id': kw.get('bitstream_id'),
'needs_firmware_update': kw.get('needs_firmware_update', False),
'status': kw.get('status'),
}
return fpga_device