Convert software image properties to byte

Convert softwareImage properties such as 'min_disk', 'min_ram' and
'size' to bytes before storing it to 'vnf_software_images' db table.
Since the size in bytes will be a large number, the type of
'min_disk' and 'min_ram' columns of 'vnf_software_images' tables has
been changed to BigInt.

Closes-Bug: #1879436
Co-Authored-By: Prashant Bhole <prash.bhole@gmail.com>
Change-Id: I3bf839783863f84f507d9b15c6eab6250d9f5d30

api-ref/source/v1/samples/vnf_packages/vnf-packages-show-response.json

@@ -10,16 +10,16 @@
 
          },
          "id":"VirtualStorage",
-         "size":2,
+         "size":2000000000, // unit for 'size` is always in Bytes
          "name":"VrtualStorage",
          "checksum":{
             "hash":"b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d",
             "algorithm":"sha-256"
          },
-         "minDisk":2,
+         "minDisk":2000000000, // unit for 'minDisk' is always in Bytes
          "version":"0.4.0",
          "provider":"provider",
-         "minRam":8192,
+         "minRam":8192000000, // unit for 'minRam' is always in Bytes
          "containerFormat":"bare"
       },
       {
@@ -29,13 +29,13 @@
 
          },
          "id":"VDU1",
-         "size":1,
+         "size":1000000000,
          "name":"Software of VDU1",
          "checksum":{
             "hash":"b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d",
             "algorithm":"sha-256"
          },
-         "minDisk":1,
+         "minDisk":1000000000,
          "version":"0.4.0",
          "provider":"provider",
          "minRam":0,

releasenotes/notes/store-software-image-prop-in-bytes-d517fee13c70d7a0.yaml

@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Fixes `bug 1879436`_. Users who build VNF packages can specify software
+    image properties like `min_disk`, `min_ram` and `size` in different units
+    as mentioned in section 3.2.6.4  of `TOSCA Simple Profile in YAML Version 1.2`_ template.
+    These property values are converted from various units to ``byte`` unit and
+    returned in `GET /vnfpkgm/v1/vnf_packages/{vnf_package_id}` API response.
+
+    .. note:: For old vnf packages, the software image properties are not converted
+            into ``byte`` unit.
+
+    .. _TOSCA Simple Profile in YAML Version 1.2: http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.2/csprd01/TOSCA-Simple-Profile-YAML-v1.2-csprd01.html
+
+    .. _bug 1879436: https://bugs.launchpad.net/tacker/+bug/1879436

tacker/common/csar_utils.py

@@ -26,6 +26,7 @@ import zipfile
 from oslo_log import log as logging
 from oslo_utils import encodeutils
 from oslo_utils import excutils
+from tacker.common import utils
 from toscaparser.prereq.csar import CSAR
 from toscaparser.tosca_template import ToscaTemplate
 
@@ -131,12 +132,30 @@ def _get_software_image(custom_defs, nodetemplate_name, node_tpl):
                 {'software_image_id': nodetemplate_name,
                 'image_path': image_path})
             sw_image_data = properties['sw_image_data']
+            _convert_software_images_prop_to_fixed_unit(sw_image_data)
+
             if 'metadata' in sw_image_artifact:
                 sw_image_data.update({'metadata':
                     sw_image_artifact['metadata']})
             return sw_image_data
 
 
+def _convert_software_images_prop_to_fixed_unit(sw_image_data):
+    """Update values of 'min_disk', 'min_ram' and 'size' to Bytes.
+
+    Since, the units like MB/MiB/GB/GiB is not stored in the database, we need
+    to convert 'min_disk', 'min_ram' and 'size' values to fixed unit before
+    saving it in database. Here its converting 'min_disk', 'min_ram', and
+    'size' values to Bytes.
+    """
+    for attribute in ['min_disk', 'min_ram', 'size']:
+        if sw_image_data.get(attribute):
+            updated_value = utils.MemoryUnit.convert_unit_size_to_num(
+                sw_image_data.get(attribute),
+                unit='B')
+            sw_image_data[attribute] = updated_value
+
+
 def _populate_flavour_data(tosca):
     flavours = []
     if tosca.nested_tosca_templates_with_topology:

tacker/common/utils.py

@@ -95,7 +95,8 @@ def find_config_file(options, config_file):
     * Search for the configuration files via common cfg directories
     :retval Full path to config file, or None if no config file found
     """
-    fix_path = lambda p: os.path.abspath(os.path.expanduser(p))  # noqa: E731
+    def fix_path(p):
+        return os.path.abspath(os.path.expanduser(p))
     if options.get('config_file'):
         if os.path.exists(options['config_file']):
             return fix_path(options['config_file'])
@@ -588,8 +589,8 @@ class MemoryUnit(object):
             unit = MemoryUnit.UNIT_SIZE_DEFAULT
             LOG.info(_('A memory unit is not provided for size; using the '
                        'default unit %(default)s.') % {'default': 'B'})
-        regex = re.compile(r'(\d*)\s*(\w*)')
+        result = re.sub(r'\s+', ' ', size).split(' ')
-        result = regex.match(str(size)).groups()
+        if len(result) == 2:
             if result[1]:
                 unit_size = MemoryUnit.validate_unit(result[1])
                 converted = int(str_to_num(result[0]) *
@@ -600,7 +601,14 @@ class MemoryUnit(object):
                            '%(unit)s.') % {'size': size,
                          'num': converted, 'unit': unit})
             else:
-            converted = (str_to_num(result[0]))
+                msg = _('Size is not given for software image data.')
+                LOG.error(msg)
+                raise ValueError(msg)
+        else:
+            msg = _('Error while converting unit "{0}" to number.'
+                    ).format(size)
+            LOG.error(msg)
+            raise ValueError(msg)
         return converted
 
     @staticmethod

tacker/conductor/conductor_server.py

@@ -371,13 +371,12 @@ class Conductor(manager.Manager):
             vnf_sw_image.container_format = sw_image.get('container_format')
             vnf_sw_image.disk_format = sw_image.get('disk_format')
             if sw_image.get('min_ram'):
-                min_ram = sw_image.get('min_ram')
+                vnf_sw_image.min_ram = sw_image.get('min_ram')
-                vnf_sw_image.min_ram = int(min_ram.split()[0])
             else:
                 vnf_sw_image.min_ram = 0
-        vnf_sw_image.min_disk = int(sw_image.get('min_disk').split()[0])
+        vnf_sw_image.min_disk = sw_image.get('min_disk')
-        vnf_sw_image.size = int(sw_image.get('size').split()[0])
+        vnf_sw_image.size = sw_image.get('size')
-        vnf_sw_image.image_path = sw_image['image_path']
+        vnf_sw_image.image_path = ''
         vnf_sw_image.software_image_id = sw_image['software_image_id']
         vnf_sw_image.metadata = sw_image.get('metadata', dict())
         vnf_sw_image.create()

tacker/db/migration/alembic_migrations/versions/329cd1619d41_alter_vnf_software_images.py

@@ -0,0 +1,38 @@
+# Copyright 2020 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""alter min_ram, min_disk columns of vnf_software_images
+
+Revision ID: 329cd1619d41
+Revises: d2e39e01d540
+Create Date: 2020-05-28 03:54:52.871841
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '329cd1619d41'
+down_revision = '329cd1619d41'
+
+
+def upgrade(active_plugins=None, options=None):
+    op.alter_column('vnf_software_images',
+                    'min_disk',
+                    type_=sa.BigInteger)
+    op.alter_column('vnf_software_images',
+                    'min_ram',
+                    type_=sa.BigInteger)

tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml

@@ -88,8 +88,8 @@ topology_template:
             hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
           container_format: bare
           disk_format: qcow2
-          min_disk: 1 GB
+          min_disk: 1000 MB
-          size: 1 GB
+          size: 1.75 GiB
 
       artifacts:
         sw_image:
@@ -141,9 +141,9 @@ topology_template:
             hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
           container_format: bare
           disk_format: qcow2
-          min_disk: 2 GB
+          min_disk: 2000 MB
-          min_ram: 8192 MB
+          min_ram: 8192.5 MiB
-          size: 2 GB
+          size: 2000 MB
       artifacts:
         sw_image:
           type: tosca.artifacts.nfv.SwImage

tacker/tests/etc/samples/etsi/nfv/sample_vnfpkg_tosca_vnfd/Definitions/helloworld3_df_simple.yaml

@@ -88,8 +88,8 @@ topology_template:
             hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
           container_format: bare
           disk_format: qcow2
-          min_disk: 1 GB
+          min_disk: 1000 MB
-          size: 1 GB
+          size: 1.75 GiB
 
       artifacts:
         sw_image:
@@ -141,9 +141,9 @@ topology_template:
             hash: b9c3036539fd7a5f87a1bf38eb05fdde8b556a1a7e664dbeda90ed3cd74b4f9d
           container_format: bare
           disk_format: qcow2
-          min_disk: 2 GB
+          min_disk: 2000 MB
-          min_ram: 8192 MB
+          min_ram: 8192.5 MiB
-          size: 2 GB
+          size: 2000 MB
       artifacts:
         sw_image:
           type: tosca.artifacts.nfv.SwImage

tacker/tests/unit/common/test_csar_utils.py

@@ -33,6 +33,9 @@ class TestCSARUtils(testtools.TestCase):
         super(TestCSARUtils, self).setUp()
         self.context = context.get_admin_context()
 
+    def _get_csar_file_path(self, file_name):
+        return os.path.join("./tacker/tests/etc/samples", file_name)
+
     @mock.patch('tacker.common.csar_utils.extract_csar_zip_file')
     def test_load_csar_data(self, mock_extract_csar_zip_file):
         file_path, _ = utils.create_csar_with_unique_vnfd_id(
@@ -441,3 +444,23 @@ class TestCSARUtils(testtools.TestCase):
                ' is added more than one time for node VDU1.')
         self.assertEqual(msg, exc.format_message())
         os.remove(zip_name)
+
+    @mock.patch('tacker.common.csar_utils.extract_csar_zip_file')
+    def test_load_csar_data_with_unit_conversion(
+            self, mock_extract_csar_zip_file):
+        file_path, _ = utils.create_csar_with_unique_vnfd_id(
+            './tacker/tests/etc/samples/etsi/nfv/sample_vnfpkg_tosca_vnfd')
+        self.addCleanup(os.remove, file_path)
+        vnf_data, flavours, vnf_artifact = csar_utils.load_csar_data(
+            self.context, constants.UUID, file_path)
+        self.assertEqual(vnf_data['descriptor_version'], '1.0')
+        self.assertEqual(vnf_data['vnfm_info'], ['Tacker'])
+        self.assertEqual(flavours[0]['flavour_id'], 'simple')
+        self.assertIsNotNone(flavours[0]['sw_images'])
+        # 'size', 'min_disk' and 'min_ram' values from sample VNFD will
+        # be converted to Bytes
+        self.assertEqual(flavours[0]['sw_images'][0]['min_disk'], 1000000000)
+        self.assertEqual(flavours[0]['sw_images'][0]['size'], 1879048192)
+        self.assertEqual(flavours[0]['sw_images'][1]['min_disk'], 2000000000)
+        self.assertEqual(flavours[0]['sw_images'][1]['size'], 2000000000)
+        self.assertEqual(flavours[0]['sw_images'][1]['min_ram'], 8590458880)