Add ability to filter by volume_glance_metadata

This feature allows users to more conveniently query volume details by
filtering the volume list by certain image metadata.
For example, users can query a specific bootable volume quickly
filtering by image_name or other glance metadata.

APIImpact
1. User can use glance metadata to filter volume detail in cinder api.
   The query url is like this:
   "volumes/detail?glance_metadata={"image_name":"xxx"}"

2. Since microversion is implemented in M, this change will add a new
   version "3.4".

DocImpact
1.Operator would need to add glance_metadata to 'query_volume_filters'
option for new functionality to work.

Change-Id: I1d276d93ad5e799401b48d2234e61c28a3aaf790
Implements: blueprint support-volume-glance-metadata-query

cinder/api/openstack/api_version_request.py

@@ -50,6 +50,7 @@ REST_API_VERSION_HISTORY = """
     * 3.2 - Bootable filters in volume GET call no longer treats all values
             passed to it as true.
     * 3.3 - Add user messages APIs.
+    * 3.4 - Adds glance_metadata filter to list/detail volumes in _get_volumes.
 
 """
 
@@ -58,7 +59,7 @@ REST_API_VERSION_HISTORY = """
 # minimum version of the API supported.
 # Explicitly using /v1 or /v2 enpoints will still work
 _MIN_API_VERSION = "3.0"
-_MAX_API_VERSION = "3.3"
+_MAX_API_VERSION = "3.4"
 _LEGACY_API_VERSION1 = "1.0"
 _LEGACY_API_VERSION2 = "2.0"
 
@@ -128,7 +129,7 @@ class APIVersionRequest(utils.ComparableMixin):
                             method.end_version,
                             method.experimental)
 
-    def matches(self, min_version, max_version, experimental=False):
+    def matches(self, min_version, max_version=None, experimental=False):
         """Compares this version to the specified min/max range.
 
         Returns whether the version object represents a version

cinder/api/openstack/rest_api_version_history.rst

@@ -55,3 +55,8 @@ user documentation.
 3.3
 ---
   Added /messages API.
+
+3.4
+---
+  Added the filter parameters ``glance_metadata`` to
+  list/detail volumes requests.

cinder/api/v2/volumes.py

@@ -97,6 +97,9 @@ class VolumeController(wsgi.Controller):
         sort_keys, sort_dirs = common.get_sort_params(params)
         filters = params
 
+        # NOTE(wanghao): Always removing glance_metadata since we support it
+        # only in API version >= 3.4.
+        filters.pop('glance_metadata', None)
         utils.remove_invalid_filter_options(context,
                                             filters,
                                             self._get_volume_filter_options())

cinder/api/v3/volumes.py

@@ -22,20 +22,25 @@ from cinder import utils
 class VolumeController(volumes_v2.VolumeController):
     """The Volumes API controller for the OpenStack API V3."""
 
+    def __init__(self, ext_mgr):
+        super(VolumeController, self).__init__(volumes_v2.VolumeController)
+
     def _get_volumes(self, req, is_detail):
         """Returns a list of volumes, transformed through view builder."""
 
         context = req.environ['cinder.context']
+        req_version = req.api_version_request
 
         params = req.params.copy()
         marker, limit, offset = common.get_pagination_params(params)
         sort_keys, sort_dirs = common.get_sort_params(params)
         filters = params
 
-        utils.remove_invalid_filter_options(context,
+        if req_version.matches(None, "3.3"):
-                                            filters,
+            filters.pop('glance_metadata', None)
-                                            self._get_volume_filter_options())
 
+        utils.remove_invalid_filter_options(context, filters,
+                                            self._get_volume_filter_options())
         # NOTE(thingee): v2 API allows name instead of display_name
         if 'name' in sort_keys:
             sort_keys[sort_keys.index('name')] = 'display_name'

cinder/db/sqlalchemy/api.py

@@ -1694,10 +1694,10 @@ def _process_volume_filters(query, filters):
     # Apply exact match filters for everything else, ensure that the
     # filter value exists on the model
     for key in filters.keys():
-        # metadata is unique, must be a dict
+        # metadata/glance_metadata is unique, must be a dict
-        if key == 'metadata':
+        if key in ('metadata', 'glance_metadata'):
             if not isinstance(filters[key], dict):
-                LOG.debug("'metadata' filter value is not valid.")
+                LOG.debug("'%s' filter value is not valid.", key)
                 return None
             continue
         try:
@@ -1727,6 +1727,11 @@ def _process_volume_filters(query, filters):
             for k, v in value.items():
                 query = query.filter(or_(col_attr.any(key=k, value=v),
                                          col_ad_attr.any(key=k, value=v)))
+        elif key == 'glance_metadata':
+            # use models.Volume.volume_glance_metadata as column attribute key.
+            col_gl_attr = models.Volume.volume_glance_metadata
+            for k, v in value.items():
+                query = query.filter(col_gl_attr.any(key=k, value=v))
         elif isinstance(value, (list, tuple, set, frozenset)):
             # Looking for values in a list; apply to query directly
             column_attr = getattr(models.Volume, key)

cinder/tests/unit/api/v3/test_volumes.py

@@ -1,3 +1,5 @@
+# Copyright 2016 OpenStack Foundation
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -13,11 +15,13 @@
 
 
 import mock
+from oslo_config import cfg
 
 from cinder.api import extensions
 from cinder.api.openstack import api_version_request as api_version
 from cinder.api.v3 import volumes
 from cinder import context
+from cinder import db
 from cinder import test
 from cinder.tests.unit.api import fakes
 from cinder.tests.unit import fake_constants as fake
@@ -26,9 +30,10 @@ from cinder.volume.api import API as vol_get
 
 version_header_name = 'OpenStack-API-Version'
 
+CONF = cfg.CONF
+
 
 class VolumeApiTest(test.TestCase):
-
     def setUp(self):
         super(VolumeApiTest, self).setUp()
         self.ext_mgr = extensions.ExtensionManager()
@@ -70,3 +75,41 @@ class VolumeApiTest(test.TestCase):
             filters = req.params.copy()
 
             volume_get.assert_called_with(filters, True)
+
+    def _create_volume_with_glance_metadata(self):
+        vol1 = db.volume_create(self.ctxt, {'display_name': 'test1',
+                                            'project_id':
+                                            self.ctxt.project_id})
+        db.volume_glance_metadata_create(self.ctxt, vol1.id, 'image_name',
+                                         'imageTestOne')
+        vol2 = db.volume_create(self.ctxt, {'display_name': 'test2',
+                                            'project_id':
+                                            self.ctxt.project_id})
+        db.volume_glance_metadata_create(self.ctxt, vol2.id, 'image_name',
+                                         'imageTestTwo')
+        db.volume_glance_metadata_create(self.ctxt, vol2.id, 'disk_format',
+                                         'qcow2')
+        return [vol1, vol2]
+
+    def test_volume_index_filter_by_glance_metadata(self):
+        vols = self._create_volume_with_glance_metadata()
+        req = fakes.HTTPRequest.blank("/v3/volumes?glance_metadata="
+                                      "{'image_name': 'imageTestOne'}")
+        req.headers["OpenStack-API-Version"] = "volume 3.4"
+        req.api_version_request = api_version.APIVersionRequest('3.4')
+        req.environ['cinder.context'] = self.ctxt
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(1, len(volumes))
+        self.assertEqual(vols[0].id, volumes[0]['id'])
+
+    def test_volume_index_filter_by_glance_metadata_in_unsupport_version(self):
+        self._create_volume_with_glance_metadata()
+        req = fakes.HTTPRequest.blank("/v3/volumes?glance_metadata="
+                                      "{'image_name': 'imageTestOne'}")
+        req.headers["OpenStack-API-Version"] = "volume 3.0"
+        req.api_version_request = api_version.APIVersionRequest('3.0')
+        req.environ['cinder.context'] = self.ctxt
+        res_dict = self.controller.index(req)
+        volumes = res_dict['volumes']
+        self.assertEqual(2, len(volumes))

cinder/tests/unit/test_db_api.py

@@ -1180,6 +1180,44 @@ class DBAPIVolumeTestCase(BaseTest):
                                                       'deleted', 'deleted_at',
                                                       'updated_at'])
 
+    def _create_volume_with_image_metadata(self):
+        vol1 = db.volume_create(self.ctxt, {'display_name': 'test1'})
+        db.volume_glance_metadata_create(self.ctxt, vol1.id, 'image_name',
+                                         'imageTestOne')
+        db.volume_glance_metadata_create(self.ctxt, vol1.id, 'test_image_key',
+                                         'test_image_value')
+        vol2 = db.volume_create(self.ctxt, {'display_name': 'test2'})
+        db.volume_glance_metadata_create(self.ctxt, vol2.id, 'image_name',
+                                         'imageTestTwo')
+        db.volume_glance_metadata_create(self.ctxt, vol2.id, 'disk_format',
+                                         'qcow2')
+        return [vol1, vol2]
+
+    def test_volume_get_all_by_image_name_and_key(self):
+        vols = self._create_volume_with_image_metadata()
+        filters = {'glance_metadata': {'image_name': 'imageTestOne',
+                                       'test_image_key': 'test_image_value'}}
+        volumes = db.volume_get_all(self.ctxt, None, None, ['created_at'],
+                                    ['desc'], filters=filters)
+        self._assertEqualListsOfObjects([vols[0]], volumes)
+
+    def test_volume_get_all_by_image_name_and_disk_format(self):
+        vols = self._create_volume_with_image_metadata()
+        filters = {'glance_metadata': {'image_name': 'imageTestTwo',
+                                       'disk_format': 'qcow2'}}
+        volumes = db.volume_get_all(self.ctxt, None, None, ['created_at'],
+                                    ['desc'], filters=filters)
+        self._assertEqualListsOfObjects([vols[1]], volumes)
+
+    def test_volume_get_all_by_invalid_image_metadata(self):
+        # Test with invalid image metadata
+        self._create_volume_with_image_metadata()
+        filters = {'glance_metadata': {'invalid_key': 'invalid_value',
+                                       'test_image_key': 'test_image_value'}}
+        volumes = db.volume_get_all(self.ctxt, None, None, ['created_at'],
+                                    ['desc'], filters=filters)
+        self._assertEqualListsOfObjects([], volumes)
+
 
 class DBAPISnapshotTestCase(BaseTest):
 

releasenotes/notes/support-volume-glance-metadata-query-866b9e3beda2cd55.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Added support for querying volumes filtered by glance metadata key/value
+    using 'glance_metadata' optional URL parameter.
+    For example, "volumes/detail?glance_metadata={"image_name":"xxx"}".