Convert software image properties to byte

Convert softwareImage properties such as 'min_disk', 'min_ram' and
'size' to bytes before storing it to 'vnf_software_images' db table.
Since the size in bytes will be a large number, the type of
'min_disk' and 'min_ram' columns of 'vnf_software_images' tables has
been changed to BigInt.

Closes-Bug: #1879436
Co-Authored-By: Prashant Bhole <prash.bhole@gmail.com>
Change-Id: I3bf839783863f84f507d9b15c6eab6250d9f5d30
This commit is contained in:
Shubham 2020-05-07 11:43:44 +05:30 committed by Keiko Kuriu
parent f37f0ddfca
commit 01cf3efd4c
10 changed files with 136 additions and 34 deletions

View File

@ -10,16 +10,16 @@
},
"id":"VirtualStorage",
"size":2,
"size":2000000000, // unit for 'size` is always in Bytes
"name":"VrtualStorage",
"checksum":{
"hash":"b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d",
"algorithm":"sha-256"
},
"minDisk":2,
"minDisk":2000000000, // unit for 'minDisk' is always in Bytes
"version":"0.4.0",
"provider":"provider",
"minRam":8192,
"minRam":8192000000, // unit for 'minRam' is always in Bytes
"containerFormat":"bare"
},
{
@ -29,13 +29,13 @@
},
"id":"VDU1",
"size":1,
"size":1000000000,
"name":"Software of VDU1",
"checksum":{
"hash":"b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d",
"algorithm":"sha-256"
},
"minDisk":1,
"minDisk":1000000000,
"version":"0.4.0",
"provider":"provider",
"minRam":0,

View File

@ -0,0 +1,15 @@
---
fixes:
- |
Fixes `bug 1879436`_. Users who build VNF packages can specify software
image properties like `min_disk`, `min_ram` and `size` in different units
as mentioned in section 3.2.6.4 of `TOSCA Simple Profile in YAML Version 1.2`_ template.
These property values are converted from various units to ``byte`` unit and
returned in `GET /vnfpkgm/v1/vnf_packages/{vnf_package_id}` API response.
.. note:: For old vnf packages, the software image properties are not converted
into ``byte`` unit.
.. _TOSCA Simple Profile in YAML Version 1.2: http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.2/csprd01/TOSCA-Simple-Profile-YAML-v1.2-csprd01.html
.. _bug 1879436: https://bugs.launchpad.net/tacker/+bug/1879436

View File

@ -26,6 +26,7 @@ import zipfile
from oslo_log import log as logging
from oslo_utils import encodeutils
from oslo_utils import excutils
from tacker.common import utils
from toscaparser.prereq.csar import CSAR
from toscaparser.tosca_template import ToscaTemplate
@ -131,12 +132,30 @@ def _get_software_image(custom_defs, nodetemplate_name, node_tpl):
{'software_image_id': nodetemplate_name,
'image_path': image_path})
sw_image_data = properties['sw_image_data']
_convert_software_images_prop_to_fixed_unit(sw_image_data)
if 'metadata' in sw_image_artifact:
sw_image_data.update({'metadata':
sw_image_artifact['metadata']})
return sw_image_data
def _convert_software_images_prop_to_fixed_unit(sw_image_data):
"""Update values of 'min_disk', 'min_ram' and 'size' to Bytes.
Since, the units like MB/MiB/GB/GiB is not stored in the database, we need
to convert 'min_disk', 'min_ram' and 'size' values to fixed unit before
saving it in database. Here its converting 'min_disk', 'min_ram', and
'size' values to Bytes.
"""
for attribute in ['min_disk', 'min_ram', 'size']:
if sw_image_data.get(attribute):
updated_value = utils.MemoryUnit.convert_unit_size_to_num(
sw_image_data.get(attribute),
unit='B')
sw_image_data[attribute] = updated_value
def _populate_flavour_data(tosca):
flavours = []
if tosca.nested_tosca_templates_with_topology:

View File

@ -95,7 +95,8 @@ def find_config_file(options, config_file):
* Search for the configuration files via common cfg directories
:retval Full path to config file, or None if no config file found
"""
fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) # noqa: E731
def fix_path(p):
return os.path.abspath(os.path.expanduser(p))
if options.get('config_file'):
if os.path.exists(options['config_file']):
return fix_path(options['config_file'])
@ -588,19 +589,26 @@ class MemoryUnit(object):
unit = MemoryUnit.UNIT_SIZE_DEFAULT
LOG.info(_('A memory unit is not provided for size; using the '
'default unit %(default)s.') % {'default': 'B'})
regex = re.compile(r'(\d*)\s*(\w*)')
result = regex.match(str(size)).groups()
if result[1]:
unit_size = MemoryUnit.validate_unit(result[1])
converted = int(str_to_num(result[0]) *
MemoryUnit.UNIT_SIZE_DICT[unit_size] *
math.pow(MemoryUnit.UNIT_SIZE_DICT
[unit], -1))
LOG.info(_('Given size %(size)s is converted to %(num)s '
'%(unit)s.') % {'size': size,
'num': converted, 'unit': unit})
result = re.sub(r'\s+', ' ', size).split(' ')
if len(result) == 2:
if result[1]:
unit_size = MemoryUnit.validate_unit(result[1])
converted = int(str_to_num(result[0]) *
MemoryUnit.UNIT_SIZE_DICT[unit_size] *
math.pow(MemoryUnit.UNIT_SIZE_DICT
[unit], -1))
LOG.info(_('Given size %(size)s is converted to %(num)s '
'%(unit)s.') % {'size': size,
'num': converted, 'unit': unit})
else:
msg = _('Size is not given for software image data.')
LOG.error(msg)
raise ValueError(msg)
else:
converted = (str_to_num(result[0]))
msg = _('Error while converting unit "{0}" to number.'
).format(size)
LOG.error(msg)
raise ValueError(msg)
return converted
@staticmethod

View File

@ -371,13 +371,12 @@ class Conductor(manager.Manager):
vnf_sw_image.container_format = sw_image.get('container_format')
vnf_sw_image.disk_format = sw_image.get('disk_format')
if sw_image.get('min_ram'):
min_ram = sw_image.get('min_ram')
vnf_sw_image.min_ram = int(min_ram.split()[0])
vnf_sw_image.min_ram = sw_image.get('min_ram')
else:
vnf_sw_image.min_ram = 0
vnf_sw_image.min_disk = int(sw_image.get('min_disk').split()[0])
vnf_sw_image.size = int(sw_image.get('size').split()[0])
vnf_sw_image.image_path = sw_image['image_path']
vnf_sw_image.min_disk = sw_image.get('min_disk')
vnf_sw_image.size = sw_image.get('size')
vnf_sw_image.image_path = ''
vnf_sw_image.software_image_id = sw_image['software_image_id']
vnf_sw_image.metadata = sw_image.get('metadata', dict())
vnf_sw_image.create()

View File

@ -0,0 +1,38 @@
# Copyright 2020 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""alter min_ram, min_disk columns of vnf_software_images
Revision ID: 329cd1619d41
Revises: d2e39e01d540
Create Date: 2020-05-28 03:54:52.871841
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '329cd1619d41'
down_revision = '329cd1619d41'
def upgrade(active_plugins=None, options=None):
op.alter_column('vnf_software_images',
'min_disk',
type_=sa.BigInteger)
op.alter_column('vnf_software_images',
'min_ram',
type_=sa.BigInteger)

View File

@ -88,8 +88,8 @@ topology_template:
hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
container_format: bare
disk_format: qcow2
min_disk: 1 GB
size: 1 GB
min_disk: 1000 MB
size: 1.75 GiB
artifacts:
sw_image:
@ -141,9 +141,9 @@ topology_template:
hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
container_format: bare
disk_format: qcow2
min_disk: 2 GB
min_ram: 8192 MB
size: 2 GB
min_disk: 2000 MB
min_ram: 8192.5 MiB
size: 2000 MB
artifacts:
sw_image:
type: tosca.artifacts.nfv.SwImage

View File

@ -88,8 +88,8 @@ topology_template:
hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
container_format: bare
disk_format: qcow2
min_disk: 1 GB
size: 1 GB
min_disk: 1000 MB
size: 1.75 GiB
artifacts:
sw_image:
@ -141,9 +141,9 @@ topology_template:
hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
container_format: bare
disk_format: qcow2
min_disk: 2 GB
min_ram: 8192 MB
size: 2 GB
min_disk: 2000 MB
min_ram: 8192.5 MiB
size: 2000 MB
artifacts:
sw_image:
type: tosca.artifacts.nfv.SwImage

View File

@ -33,6 +33,9 @@ class TestCSARUtils(testtools.TestCase):
super(TestCSARUtils, self).setUp()
self.context = context.get_admin_context()
def _get_csar_file_path(self, file_name):
return os.path.join("./tacker/tests/etc/samples", file_name)
@mock.patch('tacker.common.csar_utils.extract_csar_zip_file')
def test_load_csar_data(self, mock_extract_csar_zip_file):
file_path, _ = utils.create_csar_with_unique_vnfd_id(
@ -441,3 +444,23 @@ class TestCSARUtils(testtools.TestCase):
' is added more than one time for node VDU1.')
self.assertEqual(msg, exc.format_message())
os.remove(zip_name)
@mock.patch('tacker.common.csar_utils.extract_csar_zip_file')
def test_load_csar_data_with_unit_conversion(
self, mock_extract_csar_zip_file):
file_path, _ = utils.create_csar_with_unique_vnfd_id(
'./tacker/tests/etc/samples/etsi/nfv/sample_vnfpkg_tosca_vnfd')
self.addCleanup(os.remove, file_path)
vnf_data, flavours, vnf_artifact = csar_utils.load_csar_data(
self.context, constants.UUID, file_path)
self.assertEqual(vnf_data['descriptor_version'], '1.0')
self.assertEqual(vnf_data['vnfm_info'], ['Tacker'])
self.assertEqual(flavours[0]['flavour_id'], 'simple')
self.assertIsNotNone(flavours[0]['sw_images'])
# 'size', 'min_disk' and 'min_ram' values from sample VNFD will
# be converted to Bytes
self.assertEqual(flavours[0]['sw_images'][0]['min_disk'], 1000000000)
self.assertEqual(flavours[0]['sw_images'][0]['size'], 1879048192)
self.assertEqual(flavours[0]['sw_images'][1]['min_disk'], 2000000000)
self.assertEqual(flavours[0]['sw_images'][1]['size'], 2000000000)
self.assertEqual(flavours[0]['sw_images'][1]['min_ram'], 8590458880)

View File

@ -622,7 +622,7 @@ def get_vnfd_dict(image_path=None):
'container_format':
'fake container format',
'disk_format': 'fake disk format',
'min_disk': '1''GiB',
'min_disk': '1 ''GiB',
'name': 'fake name',
'size': 'fake size ' 'GiB',
'version': 'fake version'},