Add db support to filter vnf packages

Added db support to filter out the vnf packages as per the
filter rule specified in the `filter` query parameter.

Made use of ready-made 'sqlalchemy-filters' library which
can apply filters to the query object dynamically.

You can find few examples on home page of sqlalchemy-filters
github repository:
https://github.com/juliotrigo/sqlalchemy-filters

Change-Id: Iddf0ba7445fa7739440d333d914e675bd0782040
Implements: bp/enhance-vnf-package-support-part1

lower-constraints.txt

@@ -140,6 +140,7 @@ six==1.10.0
 snowballstemmer==1.2.1
 Sphinx==1.6.5
 sphinxcontrib-websupport==1.0.1
+sqlalchemy-filters==0.10.0
 sqlalchemy-migrate==0.11.0
 SQLAlchemy==1.3.0
 sqlparse==0.2.4

requirements.txt

@@ -15,6 +15,7 @@ keystonemiddleware>=4.17.0 # Apache-2.0
 kombu!=4.0.2,>=4.0.0 # BSD
 netaddr>=0.7.18 # BSD
 SQLAlchemy>=1.3.0 # MIT
+sqlalchemy-filters>=0.10.0
 WebOb>=1.7.1 # MIT
 python-heatclient>=1.10.0 # Apache-2.0
 python-keystoneclient>=3.8.0 # Apache-2.0

tacker/conductor/conductor_server.py

@@ -384,9 +384,11 @@ class Conductor(manager.Manager):
 
         time_duration = datetime.datetime.utcnow() - datetime.timedelta(
             seconds=CONF.vnf_package_delete_interval)
-        filters = {'deleted_at': time_duration}
+        filters = {'field': 'deleted_at', 'model': 'VnfPackage',
+                   'value': time_duration,
+                   'op': '>='}
         deleted_vnf_packages = VnfPackagesList.get_by_filters(
-            context.elevated(), read_deleted='only', **filters)
+            context.elevated(), read_deleted='only', filters=filters)
         for vnf_pack in deleted_vnf_packages:
             csar_zip_temp_path = (CONF.vnf_package.vnf_package_csar_path +
                                   vnf_pack.id)

tacker/objects/vnf_package.py

@@ -14,12 +14,14 @@
 
 from oslo_db import exception as db_exc
 from oslo_log import log as logging
+from oslo_serialization import jsonutils as json
 from oslo_utils import excutils
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
 from oslo_versionedobjects import base as ovoo_base
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql import func
+from sqlalchemy_filters import apply_filters
 
 from tacker._i18n import _
 from tacker.common import exceptions
@@ -153,15 +155,27 @@ def _vnf_package_list(context, columns_to_join=None):
 
 
 @db_api.context_manager.reader
-def _vnf_package_list_by_filters(context, read_deleted=None, **filters):
+def _vnf_package_list_by_filters(context, read_deleted=None, filters=None):
     query = api.model_query(context, models.VnfPackage,
-                            read_deleted=read_deleted, project_only=True)
+                            read_deleted=read_deleted,
-    for key, value in filters.items():
+                            project_only=True).options(joinedload('_metadata'))
-        filter_obj = getattr(models.VnfPackage, key)
+
-        if key == 'deleted_at':
+    if filters:
-            query = query.filter(filter_obj >= value)
+        # Need to join VnfDeploymentFlavour, VnfSoftwareImage and
-        else:
+        # VnfSoftwareImageMetadata db table explicitly
-            query = query.filter(filter_obj == value)
+        # only when filters contains one of the column matching
+        # from VnfSoftwareImage or VnfSoftwareImageMetadata db table.
+        filter_data = json.dumps(filters)
+        if 'VnfSoftwareImageMetadata' in filter_data:
+            query = query.join(models.VnfDeploymentFlavour).join(
+                models.VnfSoftwareImage).join(
+                models.VnfSoftwareImageMetadata)
+        elif 'VnfSoftwareImage' in filter_data:
+            query = query.join(models.VnfDeploymentFlavour).join(
+                models.VnfSoftwareImage)
+
+        query = apply_filters(query, filters)
+
     return query.all()
 
 
@@ -224,7 +238,7 @@ def _destroy_vnf_package(context, package_uuid):
 
 
 def _make_vnf_packages_list(context, vnf_package_list, db_vnf_package_list,
-                            expected_attrs):
+                            expected_attrs=None):
     vnf_package_cls = VnfPackage
 
     vnf_package_list.objects = []
@@ -289,8 +303,7 @@ class VnfPackage(base.TackerObject, base.TackerPersistentObject,
                                          expected_attrs=None):
         """Method to help with migration of extra attributes to objects."""
 
-        if expected_attrs is None:
+        expected_attrs = expected_attrs or []
-            expected_attrs = []
 
         if 'vnf_deployment_flavours' in expected_attrs:
             vnf_package._load_vnf_deployment_flavours(
@@ -465,7 +478,8 @@ class VnfPackagesList(ovoo_base.ObjectListBase, base.TackerObject):
                                        expected_attrs)
 
     @base.remotable_classmethod
-    def get_by_filters(self, context, read_deleted=None, **filters):
+    def get_by_filters(cls, context, read_deleted=None, filters=None):
-        return _vnf_package_list_by_filters(context,
+        db_vnf_packages = _vnf_package_list_by_filters(context,
                                             read_deleted=read_deleted,
-                                            **filters)
+                                            filters=filters)
+        return _make_vnf_packages_list(context, cls(), db_vnf_packages)

tacker/tests/unit/objects/test_vnf_package.py

@@ -171,7 +171,9 @@ class TestVnfPackage(SqlTestCase):
             vnf_deployment_flavour_obj.obj_load_attr, 'algorithm')
 
     def test_vnf_package_list_by_filter(self):
-        filters = {'onboarding_state': 'CREATED'}
+        filters = {'field': 'onboarding_state', 'model': 'VnfPackage',
+                   'value': 'CREATED',
+                   'op': '=='}
         vnfpkgm_list = objects.VnfPackagesList.get_by_filters(
-            self.context, **filters)
+            self.context, filters=filters)
         self.assertEqual(1, len(vnfpkgm_list))