Merge "Simple tenant usage pagination"

api-ref/source/os-simple-tenant-usage.inc

@@ -27,6 +27,8 @@ Request
   - detailed: detailed_simple_tenant_usage
   - end: end_simple_tenant_usage
   - start: start_simple_tenant_usage
+  - limit: usage_limit
+  - marker: usage_marker
 
 Response
 --------
@@ -60,20 +62,20 @@ Response
 If the ``detailed`` query parameter is not specified or
 is set to other than 1 (e.g. ``detailed=0``), the response is as follows:
 
-.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get.json
+.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json
    :language: javascript
 
 If the ``detailed`` query parameter is set to one (``detailed=1``),
 the response includes ``server_usages`` information for each tenant.
 The response is as follows:
 
-.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get-detail.json
+.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json
    :language: javascript
 
 Show Usage Statistics For Tenant
 ================================
 
-.. rest_method:: GET /os-simple-tenant-usage/{tenant_id}
+.. rest_method:: GET /os-simple-tenant-usage/v2.40/{tenant_id}
 
 Shows usage statistics for a tenant.
 
@@ -89,6 +91,8 @@ Request
   - tenant_id: tenant_id
   - end: end_simple_tenant_usage
   - start: start_simple_tenant_usage
+  - limit: usage_limit
+  - marker: usage_marker
 
 Response
 --------
@@ -119,5 +123,5 @@ Response
 
 **Example Show Usage Details For Tenant: JSON response**
 
-.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get-specific.json
+.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json
    :language: javascript

api-ref/source/parameters.yaml

@@ -717,6 +717,25 @@ tags_query:
     all tags in this list will be returned. Boolean expression in this
     case is 't1 AND t2'. Tags in query must be separated by comma.
   min_version: 2.26
+usage_limit:
+  description: |
+    Requests a page size of items. Calculate usage for the limited number of
+    instances. Use the ``limit`` parameter to make an initial limited request
+    and use the last-seen instance UUID from the response as the ``marker``
+    parameter value in a subsequent limited request.
+  in: query
+  required: false
+  type: integer
+  min_version: 2.40
+usage_marker:
+  description: |
+    The last-seen item. Use the ``limit`` parameter to make an initial limited
+    request and use the last-seen instance UUID from the response as the
+    ``marker`` parameter value in a subsequent limited request.
+  in: query
+  required: false
+  type: string
+  min_version: 2.40
 user_id_query_quota:
   description: |
     ID of user to list the quotas for.

doc/api_samples/keypairs/v2.35/keypairs-list-user2-resp.json

@@ -11,7 +11,7 @@
     ],
     "keypairs_links": [
         {
-            "href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/keypairs?user_id=user2&limit=1&marker=keypair-5d935425-31d5-48a7-a0f1-e76e9813f2c3",
+            "href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/keypairs?limit=1&marker=keypair-5d935425-31d5-48a7-a0f1-e76e9813f2c3&user_id=user2",
             "rel": "next"
         }
     ]

doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json

@@ -0,0 +1,35 @@
+{
+    "tenant_usages": [
+        {
+            "start": "2012-10-08T20:10:44.587336",
+            "stop": "2012-10-08T21:10:44.587336",
+            "tenant_id": "6f70656e737461636b20342065766572",
+            "total_hours": 1.0,
+            "total_local_gb_usage": 1.0,
+            "total_memory_mb_usage": 512.0,
+            "total_vcpus_usage": 1.0,
+            "server_usages": [
+                {
+                    "ended_at": null,
+                    "flavor": "m1.tiny",
+                    "hours": 1.0,
+                    "instance_id": "1f1deceb-17b5-4c04-84c7-e0d4499c8fe0",
+                    "local_gb": 1,
+                    "memory_mb": 512,
+                    "name": "instance-2",
+                    "started_at": "2012-10-08T20:10:44.541277",
+                    "state": "active",
+                    "tenant_id": "6f70656e737461636b20342065766572",
+                    "uptime": 3600,
+                    "vcpus": 1
+                }
+            ]
+        }
+    ],
+    "tenant_usages_links": [
+        {
+            "href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage?detailed=1&end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
+            "rel": "next"
+        }
+    ]
+}

doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json

@@ -0,0 +1,33 @@
+{
+    "tenant_usage": {
+        "server_usages": [
+            {
+                "ended_at": null,
+                "flavor": "m1.tiny",
+                "hours": 1.0,
+                "instance_id": "1f1deceb-17b5-4c04-84c7-e0d4499c8fe0",
+                "local_gb": 1,
+                "memory_mb": 512,
+                "name": "instance-2",
+                "started_at": "2012-10-08T20:10:44.541277",
+                "state": "active",
+                "tenant_id": "6f70656e737461636b20342065766572",
+                "uptime": 3600,
+                "vcpus": 1
+            }
+        ],
+        "start": "2012-10-08T20:10:44.587336",
+        "stop": "2012-10-08T21:10:44.587336",
+        "tenant_id": "6f70656e737461636b20342065766572",
+        "total_hours": 1.0,
+        "total_local_gb_usage": 1.0,
+        "total_memory_mb_usage": 512.0,
+        "total_vcpus_usage": 1.0
+    },
+    "tenant_usage_links": [
+        {
+            "href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage/6f70656e737461636b20342065766572?end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
+            "rel": "next"
+        }
+    ]
+}

doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json

@@ -0,0 +1,19 @@
+{
+    "tenant_usages": [
+        {
+            "start": "2012-10-08T21:10:44.587336",
+            "stop": "2012-10-08T22:10:44.587336",
+            "tenant_id": "6f70656e737461636b20342065766572",
+            "total_hours": 1.0,
+            "total_local_gb_usage": 1.0,
+            "total_memory_mb_usage": 512.0,
+            "total_vcpus_usage": 1.0
+        }
+    ],
+    "tenant_usages_links": [
+        {
+            "href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage?end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
+            "rel": "next"
+        }
+    ]
+}

doc/api_samples/versions/v21-version-get-resp.json

@@ -19,7 +19,7 @@
             }
         ],
         "status": "CURRENT",
-        "version": "2.39",
+        "version": "2.40",
         "min_version": "2.1",
         "updated": "2013-07-23T11:33:21Z"
     }

doc/api_samples/versions/versions-get-resp.json

@@ -22,7 +22,7 @@
                 }
             ],
             "status": "CURRENT",
-            "version": "2.39",
+            "version": "2.40",
             "min_version": "2.1",
             "updated": "2013-07-23T11:33:21Z"
         }

nova/api/openstack/api_version_request.py

@@ -96,6 +96,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
     * 2.38 - Add a condition to return HTTPBadRequest if invalid status is
              provided for listing servers.
     * 2.39 - Deprecates image-metadata proxy API
+    * 2.40 - Adds simple tenant usage pagination support.
 """
 
 # The minimum and maximum versions of the API supported
@@ -104,7 +105,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
 # Note(cyeoh): This only applies for the v2.1 API once microversions
 # support is fully merged. It does not affect the V2 API.
 _MIN_API_VERSION = "2.1"
-_MAX_API_VERSION = "2.39"
+_MAX_API_VERSION = "2.40"
 DEFAULT_API_VERSION = _MIN_API_VERSION
 
 # Almost all proxy APIs which related to network, images and baremetal

nova/api/openstack/common.py

@@ -401,7 +401,7 @@ class ViewBuilder(object):
 
     def _get_next_link(self, request, identifier, collection_name):
         """Return href string with proper limit and marker params."""
-        params = request.params.copy()
+        params = collections.OrderedDict(sorted(request.params.items()))
         params["marker"] = identifier
         prefix = self._update_compute_link_prefix(request.application_url)
         url = url_join(prefix,

nova/api/openstack/compute/simple_tenant_usage.py

@@ -21,13 +21,17 @@ import six
 import six.moves.urllib.parse as urlparse
 from webob import exc
 
+from nova.api.openstack import common
+from nova.api.openstack.compute.views import usages as usages_view
 from nova.api.openstack import extensions
 from nova.api.openstack import wsgi
+import nova.conf
 from nova import exception
 from nova.i18n import _
 from nova import objects
 from nova.policies import simple_tenant_usage as stu_policies
 
+CONF = nova.conf.CONF
 ALIAS = "os-simple-tenant-usage"
 
 
@@ -39,6 +43,9 @@ def parse_strtime(dstr, fmt):
 
 
 class SimpleTenantUsageController(wsgi.Controller):
+
+    _view_builder_class = usages_view.ViewBuilder
+
     def _hours_for(self, instance, period_start, period_stop):
         launched_at = instance.launched_at
         terminated_at = instance.terminated_at
@@ -97,14 +104,16 @@ class SimpleTenantUsageController(wsgi.Controller):
 
         return flavor_ref
 
-    def _tenant_usages_for_period(self, context, period_start,
+    def _tenant_usages_for_period(self, context, period_start, period_stop,
-                                  period_stop, tenant_id=None, detailed=True):
+                                  tenant_id=None, detailed=True, limit=None,
+                                  marker=None):
 
         instances = objects.InstanceList.get_active_by_window_joined(
                         context, period_start, period_stop, tenant_id,
-                        expected_attrs=['flavor'])
+                        expected_attrs=['flavor'], limit=limit, marker=marker)
         rval = {}
         flavors = {}
+        all_server_usages = []
 
         for instance in instances:
             info = {}
@@ -170,10 +179,11 @@ class SimpleTenantUsageController(wsgi.Controller):
                                                  info['hours'])
 
             summary['total_hours'] += info['hours']
+            all_server_usages.append(info)
             if detailed:
                 summary['server_usages'].append(info)
 
-        return rval.values()
+        return list(rval.values()), all_server_usages
 
     def _parse_datetime(self, dtstr):
         if not dtstr:
@@ -216,9 +226,31 @@ class SimpleTenantUsageController(wsgi.Controller):
         detailed = env.get('detailed', ['0'])[0] == '1'
         return (period_start, period_stop, detailed)
 
+    @wsgi.Controller.api_version("2.40")
     @extensions.expected_errors(400)
     def index(self, req):
         """Retrieve tenant_usage for all tenants."""
+        return self._index(req, links=True)
+
+    @wsgi.Controller.api_version("2.1", "2.39")  # noqa
+    @extensions.expected_errors(400)
+    def index(self, req):
+        """Retrieve tenant_usage for all tenants."""
+        return self._index(req)
+
+    @wsgi.Controller.api_version("2.40")
+    @extensions.expected_errors(400)
+    def show(self, req, id):
+        """Retrieve tenant_usage for a specified tenant."""
+        return self._show(req, id, links=True)
+
+    @wsgi.Controller.api_version("2.1", "2.39")  # noqa
+    @extensions.expected_errors(400)
+    def show(self, req, id):
+        """Retrieve tenant_usage for a specified tenant."""
+        return self._show(req, id)
+
+    def _index(self, req, links=False):
         context = req.environ['nova.context']
 
         context.can(stu_policies.POLICY_ROOT % 'list')
@@ -232,15 +264,29 @@ class SimpleTenantUsageController(wsgi.Controller):
         now = timeutils.parse_isotime(timeutils.utcnow().isoformat())
         if period_stop > now:
             period_stop = now
-        usages = self._tenant_usages_for_period(context,
-                                                period_start,
-                                                period_stop,
-                                                detailed=detailed)
-        return {'tenant_usages': usages}
 
-    @extensions.expected_errors(400)
+        marker = None
-    def show(self, req, id):
+        limit = CONF.api.max_limit
-        """Retrieve tenant_usage for a specified tenant."""
+        if links:
+            limit, marker = common.get_limit_and_marker(req)
+
+        try:
+            usages, server_usages = self._tenant_usages_for_period(
+                context, period_start, period_stop, detailed=detailed,
+                limit=limit, marker=marker)
+        except exception.MarkerNotFound as e:
+            raise exc.HTTPBadRequest(explanation=e.format_message())
+
+        tenant_usages = {'tenant_usages': usages}
+
+        if links:
+            usages_links = self._view_builder.get_links(req, server_usages)
+            if usages_links:
+                tenant_usages['tenant_usages_links'] = usages_links
+
+        return tenant_usages
+
+    def _show(self, req, id, links=False):
         tenant_id = id
         context = req.environ['nova.context']
 
@@ -256,16 +302,33 @@ class SimpleTenantUsageController(wsgi.Controller):
         now = timeutils.parse_isotime(timeutils.utcnow().isoformat())
         if period_stop > now:
             period_stop = now
-        usage = self._tenant_usages_for_period(context,
+
-                                               period_start,
+        marker = None
-                                               period_stop,
+        limit = CONF.api.max_limit
-                                               tenant_id=tenant_id,
+        if links:
-                                               detailed=True)
+            limit, marker = common.get_limit_and_marker(req)
+
+        try:
+            usage, server_usages = self._tenant_usages_for_period(
+                context, period_start, period_stop, tenant_id=tenant_id,
+                detailed=True, limit=limit, marker=marker)
+        except exception.MarkerNotFound as e:
+            raise exc.HTTPBadRequest(explanation=e.format_message())
+
         if len(usage):
             usage = list(usage)[0]
         else:
             usage = {}
-        return {'tenant_usage': usage}
+
+        tenant_usage = {'tenant_usage': usage}
+
+        if links:
+            usages_links = self._view_builder.get_links(
+                req, server_usages, tenant_id=tenant_id)
+            if usages_links:
+                tenant_usage['tenant_usage_links'] = usages_links
+
+        return tenant_usage
 
 
 class SimpleTenantUsage(extensions.V21APIExtensionBase):

nova/api/openstack/compute/views/usages.py

@@ -0,0 +1,28 @@
+# Copyright 2016 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova.api.openstack import common
+
+
+class ViewBuilder(common.ViewBuilder):
+
+    _collection_name = "os-simple-tenant-usage"
+
+    def get_links(self, request, server_usages, tenant_id=None):
+        coll_name = self._collection_name
+        if tenant_id:
+            coll_name = self._collection_name + '/{}'.format(tenant_id)
+        return self._get_collection_links(
+            request, server_usages, coll_name, 'instance_id')

nova/api/openstack/rest_api_version_history.rst

@@ -426,3 +426,22 @@ user documentation.
   option `image_property_quota` should be used to control the quota of
   image metadatas. Also, removes the `maxImageMeta` field from `os-limits`
   API response.
+
+2.40
+----
+
+  Optional parameters 'limit' and 'marker' were added to the GET
+  /os-simple-tenant-usage and GET os-simple-tenant-usage/{tenant_id}
+  requests. The aggregate usage data no longer reflects all instances for a
+  tenant, but rather just the paginated instances ordered by instance id ASC.
+  API consumers will need to stitch the aggregate data back up (add the totals)
+  if a tenant's instances span several pages.
+
+    GET /os-simple-tenant-usage?limit={limit}&marker={instance_uuid}
+    GET /os-simple-tenant-usage/{tenant_id}?limit={limit}&marker={instance_uuid}
+
+  Older versions of the `os-simple-tenant-usage` endpoints will not accept
+  these new paging query parameters, but they will start to silently limit by
+  `CONF.api.max_limit` to encourage the adoption of this new microversion,
+  and circumvent the existing possibility DoS-like usage requests on systems
+  with thousands of instances.

nova/db/api.py

@@ -765,7 +765,8 @@ def instance_get_all_by_filters_sort(context, filters, limit=None,
 
 def instance_get_active_by_window_joined(context, begin, end=None,
                                          project_id=None, host=None,
-                                         columns_to_join=None):
+                                         columns_to_join=None, limit=None,
+                                         marker=None):
     """Get instances and joins active during a certain time window.
 
     Specifying a project_id will filter for a certain project.
@@ -773,7 +774,8 @@ def instance_get_active_by_window_joined(context, begin, end=None,
     """
     return IMPL.instance_get_active_by_window_joined(context, begin, end,
                                               project_id, host,
-                                              columns_to_join=columns_to_join)
+                                              columns_to_join=columns_to_join,
+                                              limit=limit, marker=marker)
 
 
 def instance_get_all_by_host(context, host, columns_to_join=None):

nova/db/sqlalchemy/api.py

@@ -2513,7 +2513,8 @@ def process_sort_params(sort_keys, sort_dirs,
 @pick_context_manager_reader_allow_async
 def instance_get_active_by_window_joined(context, begin, end=None,
                                          project_id=None, host=None,
-                                         columns_to_join=None):
+                                         columns_to_join=None, limit=None,
+                                         marker=None):
     """Return instances and joins that were active during window."""
     query = context.session.query(models.Instance)
 
@@ -2539,6 +2540,16 @@ def instance_get_active_by_window_joined(context, begin, end=None,
     if host:
         query = query.filter_by(host=host)
 
+    if marker is not None:
+        try:
+            marker = _instance_get_by_uuid(
+                context.elevated(read_deleted='yes'), marker)
+        except exception.InstanceNotFound:
+            raise exception.MarkerNotFound(marker=marker)
+
+    query = sqlalchemyutils.paginate_query(
+        query, models.Instance, limit, ['project_id', 'uuid'], marker=marker)
+
     return _instances_fill_metadata(context, query.all(), manual_joins)
 
 

nova/objects/instance.py

@@ -1184,7 +1184,8 @@ def _make_instance_list(context, inst_list, db_inst_list, expected_attrs):
 class InstanceList(base.ObjectListBase, base.NovaObject):
     # Version 2.0: Initial Version
     # Version 2.1: Add get_uuids_by_host()
-    VERSION = '2.1'
+    # Version 2.2: Pagination for get_active_by_window_joined()
+    VERSION = '2.2'
 
     fields = {
         'objects': fields.ListOfObjectsField('Instance'),
@@ -1269,16 +1270,16 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
     @db.select_db_reader_mode
     def _db_instance_get_active_by_window_joined(
             context, begin, end, project_id, host, columns_to_join,
-            use_slave=False):
+            use_slave=False, limit=None, marker=None):
         return db.instance_get_active_by_window_joined(
             context, begin, end, project_id, host,
-            columns_to_join=columns_to_join)
+            columns_to_join=columns_to_join, limit=limit, marker=marker)
 
     @base.remotable_classmethod
     def _get_active_by_window_joined(cls, context, begin, end=None,
                                     project_id=None, host=None,
-                                    expected_attrs=None,
+                                    expected_attrs=None, use_slave=False,
-                                    use_slave=False):
+                                    limit=None, marker=None):
         # NOTE(mriedem): We need to convert the begin/end timestamp strings
         # to timezone-aware datetime objects for the DB API call.
         begin = timeutils.parse_isotime(begin)
@@ -1286,15 +1287,15 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
         db_inst_list = cls._db_instance_get_active_by_window_joined(
             context, begin, end, project_id, host,
             columns_to_join=_expected_cols(expected_attrs),
-            use_slave=use_slave)
+            use_slave=use_slave, limit=limit, marker=marker)
         return _make_instance_list(context, cls(), db_inst_list,
                                    expected_attrs)
 
     @classmethod
     def get_active_by_window_joined(cls, context, begin, end=None,
                                     project_id=None, host=None,
-                                    expected_attrs=None,
+                                    expected_attrs=None, use_slave=False,
-                                    use_slave=False):
+                                    limit=None, marker=None):
         """Get instances and joins active during a certain time window.
 
         :param:context: nova request context
@@ -1305,6 +1306,8 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
         :param:expected_attrs: list of related fields that can be joined
         in the database layer when querying for instances
         :param use_slave if True, ship this query off to a DB slave
+        :param limit: maximum number of instances to return per page
+        :param marker: last instance uuid from the previous page
         :returns: InstanceList
 
         """
@@ -1315,7 +1318,8 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
         return cls._get_active_by_window_joined(context, begin, end,
                                                 project_id, host,
                                                 expected_attrs,
-                                                use_slave=use_slave)
+                                                use_slave=use_slave,
+                                                limit=limit, marker=marker)
 
     @base.remotable_classmethod
     def get_by_security_group_id(cls, context, security_group_id):

nova/tests/functional/api_sample_tests/api_samples/keypairs/v2.35/keypairs-list-user2-resp.json.tpl

@@ -11,7 +11,7 @@
     ],
     "keypairs_links": [
         {
-            "href": "%(versioned_compute_endpoint)s/keypairs?user_id=user2&limit=1&marker=%(keypair_name)s",
+            "href": "%(versioned_compute_endpoint)s/keypairs?limit=1&marker=%(keypair_name)s&user_id=user2",
             "rel": "next"
         }
     ]

nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json.tpl

@@ -0,0 +1,35 @@
+{
+    "tenant_usages": [
+        {
+            "start": "%(strtime)s",
+            "stop": "%(strtime)s",
+            "tenant_id": "6f70656e737461636b20342065766572",
+            "total_hours": 1.0,
+            "total_local_gb_usage": 1.0,
+            "total_memory_mb_usage": 512.0,
+            "total_vcpus_usage": 1.0,
+            "server_usages": [
+                {
+                    "ended_at": null,
+                    "flavor": "m1.tiny",
+                    "hours": 1.0,
+                    "instance_id": "%(uuid)s",
+                    "local_gb": 1,
+                    "memory_mb": 512,
+                    "name": "instance-2",
+                    "started_at": "%(strtime)s",
+                    "state": "active",
+                    "tenant_id": "6f70656e737461636b20342065766572",
+                    "uptime": 3600,
+                    "vcpus": 1
+                }
+            ]
+        }
+    ],
+    "tenant_usages_links": [
+        {
+            "href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage?detailed=1&end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
+            "rel": "next"
+        }
+    ]
+}

nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json.tpl

@@ -0,0 +1,33 @@
+{
+    "tenant_usage": {
+        "server_usages": [
+            {
+                "ended_at": null,
+                "flavor": "m1.tiny",
+                "hours": 1.0,
+                "instance_id": "%(uuid)s",
+                "local_gb": 1,
+                "memory_mb": 512,
+                "name": "instance-2",
+                "started_at": "%(strtime)s",
+                "state": "active",
+                "tenant_id": "6f70656e737461636b20342065766572",
+                "uptime": 3600,
+                "vcpus": 1
+            }
+        ],
+        "start": "%(strtime)s",
+        "stop": "%(strtime)s",
+        "tenant_id": "6f70656e737461636b20342065766572",
+        "total_hours": 1.0,
+        "total_local_gb_usage": 1.0,
+        "total_memory_mb_usage": 512.0,
+        "total_vcpus_usage": 1.0
+    },
+    "tenant_usage_links": [
+        {
+            "href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage/%(tenant_id)s?end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
+            "rel": "next"
+        }
+    ]
+}

nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json.tpl

@@ -0,0 +1,19 @@
+{
+    "tenant_usages": [
+        {
+            "start": "%(strtime)s",
+            "stop": "%(strtime)s",
+            "tenant_id": "6f70656e737461636b20342065766572",
+            "total_hours": 1.0,
+            "total_local_gb_usage": 1.0,
+            "total_memory_mb_usage": 512.0,
+            "total_vcpus_usage": 1.0
+        }
+    ],
+    "tenant_usages_links": [
+        {
+            "href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage?end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
+            "rel": "next"
+        }
+    ]
+}

nova/tests/functional/api_sample_tests/api_samples/servers/server-create-req-v237.json.tpl

@@ -2,7 +2,7 @@
     "server" : {
         "accessIPv4": "%(access_ip_v4)s",
         "accessIPv6": "%(access_ip_v6)s",
-        "name" : "new-server-test",
+        "name" : "%(name)s",
         "imageRef" : "%(image_id)s",
         "flavorRef" : "1",
         "availability_zone": "nova",

nova/tests/functional/api_sample_tests/test_servers.py

@@ -46,7 +46,7 @@ class ServersSampleBase(api_sample_base.ApiSampleTestBaseV21):
                     avr.APIVersionRequest(min), avr.APIVersionRequest(max)):
                 return name
 
-    def _post_server(self, use_common_server_api_samples=True):
+    def _post_server(self, use_common_server_api_samples=True, name=None):
         # param use_common_server_api_samples: Boolean to set whether tests use
         # common sample files for server post request and response.
         # Default is True which means _get_sample_path method will fetch the
@@ -63,6 +63,7 @@ class ServersSampleBase(api_sample_base.ApiSampleTestBaseV21):
             'user_data': self.user_data,
             'uuid': '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}'
                     '-[0-9a-f]{4}-[0-9a-f]{12}',
+            'name': 'new-server-test' if name is None else name,
         }
 
         orig_value = self.__class__._use_common_server_api_samples

nova/tests/functional/api_sample_tests/test_simple_tenant_usage.py

@@ -15,6 +15,7 @@
 import datetime
 import urllib
 
+import mock
 from oslo_utils import timeutils
 
 from nova.tests.functional.api_sample_tests import test_servers
@@ -67,3 +68,59 @@ class SimpleTenantUsageSampleJsonTest(test_servers.ServersSampleBase):
                                                 urllib.urlencode(self.query)))
         self._verify_response('simple-tenant-usage-get-specific', {},
                               response, 200)
+
+
+class SimpleTenantUsageV240Test(test_servers.ServersSampleBase):
+    sample_dir = 'os-simple-tenant-usage'
+    microversion = '2.40'
+    scenarios = [('v2_40', {'api_major_version': 'v2.1'})]
+
+    def setUp(self):
+        super(SimpleTenantUsageV240Test, self).setUp()
+        self.api.microversion = self.microversion
+
+        started = timeutils.utcnow()
+        now = started + datetime.timedelta(hours=1)
+
+        timeutils.set_time_override(started)
+        with mock.patch('oslo_utils.uuidutils.generate_uuid') as mock_uuids:
+            # make uuids incrementing, so that sort order is deterministic
+            uuid_format = '1f1deceb-17b5-4c04-84c7-e0d4499c8f%02d'
+            mock_uuids.side_effect = [uuid_format % x for x in range(100)]
+            self.instance1_uuid = self._post_server(name='instance-1')
+            self.instance2_uuid = self._post_server(name='instance-2')
+            self.instance3_uuid = self._post_server(name='instance-3')
+        timeutils.set_time_override(now)
+
+        self.query = {
+            'start': str(started),
+            'end': str(now),
+            'limit': '1',
+            'marker': self.instance1_uuid,
+        }
+
+    def tearDown(self):
+        super(SimpleTenantUsageV240Test, self).tearDown()
+        timeutils.clear_time_override()
+
+    def test_get_tenants_usage(self):
+        url = 'os-simple-tenant-usage?%s'
+        response = self._do_get(url % (urllib.urlencode(self.query)))
+        template_name = 'simple-tenant-usage-get'
+        self._verify_response(template_name, {}, response, 200)
+
+    def test_get_tenants_usage_with_detail(self):
+        query = self.query.copy()
+        query.update({'detailed': 1})
+        url = 'os-simple-tenant-usage?%s'
+        response = self._do_get(url % (urllib.urlencode(query)))
+        template_name = 'simple-tenant-usage-get-detail'
+        self._verify_response(template_name, {}, response, 200)
+
+    def test_get_tenant_usage_details(self):
+        tenant_id = astb.PROJECT_ID
+        url = 'os-simple-tenant-usage/{tenant}?%s'.format(tenant=tenant_id)
+        response = self._do_get(url % (urllib.urlencode(self.query)))
+        template_name = 'simple-tenant-usage-get-specific'
+        subs = {'tenant_id': self.api.project_id}
+        self._verify_response(template_name, subs, response, 200)

nova/tests/functional/api_samples_test_base.py

@@ -410,6 +410,8 @@ class ApiSampleTestBase(integrated_helpers._IntegratedTestBase):
         text = r'(\\"|[^"])*'
         isotime_re = '\d{4}-[0,1]\d-[0-3]\dT\d{2}:\d{2}:\d{2}Z'
         strtime_re = '\d{4}-[0,1]\d-[0-3]\dT\d{2}:\d{2}:\d{2}\.\d{6}'
+        strtime_url_re = ('\d{4}-[0,1]\d-[0-3]\d'
+                          '\+\d{2}\%3A\d{2}\%3A\d{2}\.\d{6}')
         xmltime_re = ('\d{4}-[0,1]\d-[0-3]\d '
                       '\d{2}:\d{2}:\d{2}'
                       '(\.\d{6})?(\+00:00)?')
@@ -419,6 +421,7 @@ class ApiSampleTestBase(integrated_helpers._IntegratedTestBase):
         return {
             'isotime': isotime_re,
             'strtime': strtime_re,
+            'strtime_url': strtime_url_re,
             'strtime_or_none': r'None|%s' % strtime_re,
             'xmltime': xmltime_re,
             'password': '[0-9a-zA-Z]{1,12}',

nova/tests/unit/api/openstack/compute/test_simple_tenant_usage.py

@@ -24,6 +24,7 @@ import webob
 from nova.api.openstack.compute import simple_tenant_usage as \
     simple_tenant_usage_v21
 from nova.compute import vm_states
+import nova.conf
 from nova import context
 from nova import exception
 from nova import objects
@@ -32,6 +33,10 @@ from nova import test
 from nova.tests.unit.api.openstack import fakes
 from nova.tests import uuidsentinel as uuids
 
+
+CONF = nova.conf.CONF
+
+
 SERVERS = 5
 TENANTS = 2
 HOURS = 24
@@ -88,16 +93,18 @@ def _fake_instance(start, end, instance_id, tenant_id,
 @classmethod
 def fake_get_active_by_window_joined(cls, context, begin, end=None,
                                      project_id=None, host=None,
-                                     expected_attrs=None, use_slave=False):
+                                     expected_attrs=None, use_slave=False,
+                                     limit=None, marker=None):
     return objects.InstanceList(objects=[
         _fake_instance(START, STOP, x,
-                       project_id or 'faketenant_%s' % (x / SERVERS))
+                       project_id or 'faketenant_%s' % (x // SERVERS))
         for x in range(TENANTS * SERVERS)])
 
 
 @mock.patch('nova.objects.InstanceList.get_active_by_window_joined',
             fake_get_active_by_window_joined)
 class SimpleTenantUsageTestV21(test.TestCase):
+    version = '2.1'
     policy_rule_prefix = "os_compute_api:os-simple-tenant-usage"
     controller = simple_tenant_usage_v21.SimpleTenantUsageController()
 
@@ -113,11 +120,16 @@ class SimpleTenantUsageTestV21(test.TestCase):
                                                       'faketenant_1',
                                                        is_admin=False)
 
-    def _test_verify_index(self, start, stop):
+    def _test_verify_index(self, start, stop, limit=None):
-        req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
+        url = '?start=%s&end=%s'
-                    (start.isoformat(), stop.isoformat()))
+        if limit:
+            url += '&limit=%s' % (limit)
+        req = fakes.HTTPRequest.blank(url %
+                    (start.isoformat(), stop.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.admin_context
         res_dict = self.controller.index(req)
+
         usages = res_dict['tenant_usages']
         for i in range(TENANTS):
             self.assertEqual(SERVERS * HOURS, int(usages[i]['total_hours']))
@@ -129,6 +141,12 @@ class SimpleTenantUsageTestV21(test.TestCase):
                              int(usages[i]['total_vcpus_usage']))
             self.assertFalse(usages[i].get('server_usages'))
 
+        if limit:
+            self.assertIn('tenant_usages_links', res_dict)
+            self.assertEqual('next', res_dict['tenant_usages_links'][0]['rel'])
+        else:
+            self.assertNotIn('tenant_usages_links', res_dict)
+
     def test_verify_index(self):
         self._test_verify_index(START, STOP)
 
@@ -145,7 +163,8 @@ class SimpleTenantUsageTestV21(test.TestCase):
 
     def _get_tenant_usages(self, detailed=''):
         req = fakes.HTTPRequest.blank('?detailed=%s&start=%s&end=%s' %
-                    (detailed, START.isoformat(), STOP.isoformat()))
+                    (detailed, START.isoformat(), STOP.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.admin_context
 
         # Make sure that get_active_by_window_joined is only called with
@@ -155,8 +174,8 @@ class SimpleTenantUsageTestV21(test.TestCase):
 
         def fake_get_active_by_window_joined(context, begin, end=None,
                                     project_id=None, host=None,
-                                    expected_attrs=None,
+                                    expected_attrs=None, use_slave=False,
-                                    use_slave=False):
+                                    limit=None, marker=None):
             self.assertEqual(['flavor'], expected_attrs)
             return orig_get_active_by_window_joined(context, begin, end,
                                                     project_id, host,
@@ -186,12 +205,15 @@ class SimpleTenantUsageTestV21(test.TestCase):
         for i in range(TENANTS):
             self.assertIsNone(usages[i].get('server_usages'))
 
-    def _test_verify_show(self, start, stop):
+    def _test_verify_show(self, start, stop, limit=None):
         tenant_id = 1
-        req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
+        url = '?start=%s&end=%s'
-                    (start.isoformat(), stop.isoformat()))
+        if limit:
+            url += '&limit=%s' % (limit)
+        req = fakes.HTTPRequest.blank(url %
+                    (start.isoformat(), stop.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.user_context
-
         res_dict = self.controller.show(req, tenant_id)
 
         usage = res_dict['tenant_usage']
@@ -207,9 +229,16 @@ class SimpleTenantUsageTestV21(test.TestCase):
             self.assertEqual(HOURS, int(servers[j]['hours']))
             self.assertIn(servers[j]['instance_id'], server_uuids)
 
+        if limit:
+            self.assertIn('tenant_usage_links', res_dict)
+            self.assertEqual('next', res_dict['tenant_usage_links'][0]['rel'])
+        else:
+            self.assertNotIn('tenant_usage_links', res_dict)
+
     def test_verify_show_cannot_view_other_tenant(self):
         req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
-                    (START.isoformat(), STOP.isoformat()))
+                    (START.isoformat(), STOP.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.alt_user_context
 
         rules = {
@@ -227,20 +256,23 @@ class SimpleTenantUsageTestV21(test.TestCase):
     def test_get_tenants_usage_with_bad_start_date(self):
         future = NOW + datetime.timedelta(hours=HOURS)
         req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
-                    (future.isoformat(), NOW.isoformat()))
+                    (future.isoformat(), NOW.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.user_context
         self.assertRaises(webob.exc.HTTPBadRequest,
                           self.controller.show, req, 'faketenant_0')
 
     def test_get_tenants_usage_with_invalid_start_date(self):
         req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
-                    ("xxxx", NOW.isoformat()))
+                    ("xxxx", NOW.isoformat()),
+                    version=self.version)
         req.environ['nova.context'] = self.user_context
         self.assertRaises(webob.exc.HTTPBadRequest,
                           self.controller.show, req, 'faketenant_0')
 
     def _test_get_tenants_usage_with_one_date(self, date_url_param):
-        req = fakes.HTTPRequest.blank('?%s' % date_url_param)
+        req = fakes.HTTPRequest.blank('?%s' % date_url_param,
+                                      version=self.version)
         req.environ['nova.context'] = self.user_context
         res = self.controller.show(req, 'faketenant_0')
         self.assertIn('tenant_usage', res)
@@ -254,6 +286,83 @@ class SimpleTenantUsageTestV21(test.TestCase):
             'start=%s' % (NOW - datetime.timedelta(5)).isoformat())
 
 
+@mock.patch('nova.objects.InstanceList.get_active_by_window_joined',
+            fake_get_active_by_window_joined)
+class SimpleTenantUsageTestV40(SimpleTenantUsageTestV21):
+    version = '2.40'
+
+    def test_next_links_show(self):
+        self._test_verify_show(START, STOP, limit=SERVERS * TENANTS)
+
+    def test_next_links_index(self):
+        self._test_verify_index(START, STOP, limit=SERVERS * TENANTS)
+
+
+class SimpleTenantUsageLimitsTestV21(test.TestCase):
+    version = '2.1'
+
+    def setUp(self):
+        super(SimpleTenantUsageLimitsTestV21, self).setUp()
+        self.controller = simple_tenant_usage_v21.SimpleTenantUsageController()
+        self.tenant_id = 1
+
+    def _get_request(self, url):
+        url = url % (START.isoformat(), STOP.isoformat())
+        return fakes.HTTPRequest.blank(url, version=self.version)
+
+    def assert_limit(self, mock_get, limit):
+        mock_get.assert_called_once_with(
+            mock.ANY, mock.ANY, mock.ANY, mock.ANY, expected_attrs=['flavor'],
+            limit=1000, marker=None)
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_limit_defaults_to_conf_max_limit_show(self, mock_get):
+        req = self._get_request('?start=%s&end=%s')
+        self.controller.show(req, self.tenant_id)
+        self.assert_limit(mock_get, CONF.api.max_limit)
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_limit_defaults_to_conf_max_limit_index(self, mock_get):
+        req = self._get_request('?start=%s&end=%s')
+        self.controller.index(req)
+        self.assert_limit(mock_get, CONF.api.max_limit)
+
+
+class SimpleTenantUsageLimitsTestV240(SimpleTenantUsageLimitsTestV21):
+    version = '2.40'
+
+    def assert_limit_and_marker(self, mock_get, limit, marker):
+        mock_get.assert_called_once_with(
+            mock.ANY, mock.ANY, mock.ANY, mock.ANY, expected_attrs=['flavor'],
+            limit=3, marker=marker)
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_limit_and_marker_show(self, mock_get):
+        req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
+        self.controller.show(req, self.tenant_id)
+        self.assert_limit_and_marker(mock_get, 3, 'some-marker')
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_limit_and_marker_index(self, mock_get):
+        req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
+        self.controller.index(req)
+        self.assert_limit_and_marker(mock_get, 3, 'some-marker')
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_marker_not_found_show(self, mock_get):
+        mock_get.side_effect = exception.MarkerNotFound(marker='some-marker')
+        req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
+        self.assertRaises(
+            webob.exc.HTTPBadRequest, self.controller.show, req, 1)
+
+    @mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
+    def test_marker_not_found_index(self, mock_get):
+        mock_get.side_effect = exception.MarkerNotFound(marker='some-marker')
+        req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
+        self.assertRaises(
+            webob.exc.HTTPBadRequest, self.controller.index, req)
+
+
 class SimpleTenantUsageControllerTestV21(test.TestCase):
     controller = simple_tenant_usage_v21.SimpleTenantUsageController()
 

nova/tests/unit/db/test_db_api.py

@@ -1199,6 +1199,41 @@ class SqlAlchemyDbApiTestCase(DbTestCase):
         self.assertEqual(2, len(result))
         self.assertEqual(six.text_type, type(result[0]))
 
+    @mock.patch('oslo_utils.uuidutils.generate_uuid')
+    def test_instance_get_active_by_window_joined_paging(self, mock_uuids):
+        mock_uuids.side_effect = ['BBB', 'ZZZ', 'AAA', 'CCC']
+
+        ctxt = context.get_admin_context()
+        now = datetime.datetime(2015, 10, 2)
+        self.create_instance_with_args(project_id='project-ZZZ')
+        self.create_instance_with_args(project_id='project-ZZZ')
+        self.create_instance_with_args(project_id='project-ZZZ')
+        self.create_instance_with_args(project_id='project-AAA')
+
+        # no limit or marker
+        result = sqlalchemy_api.instance_get_active_by_window_joined(
+            ctxt, begin=now, columns_to_join=[])
+        actual_uuids = [row['uuid'] for row in result]
+        self.assertEqual(['CCC', 'AAA', 'BBB', 'ZZZ'], actual_uuids)
+
+        # just limit
+        result = sqlalchemy_api.instance_get_active_by_window_joined(
+            ctxt, begin=now, columns_to_join=[], limit=2)
+        actual_uuids = [row['uuid'] for row in result]
+        self.assertEqual(['CCC', 'AAA'], actual_uuids)
+
+        # limit & marker
+        result = sqlalchemy_api.instance_get_active_by_window_joined(
+            ctxt, begin=now, columns_to_join=[], limit=2, marker='CCC')
+        actual_uuids = [row['uuid'] for row in result]
+        self.assertEqual(['AAA', 'BBB'], actual_uuids)
+
+        # unknown marker
+        self.assertRaises(
+            exception.MarkerNotFound,
+            sqlalchemy_api.instance_get_active_by_window_joined,
+            ctxt, begin=now, columns_to_join=[], limit=2, marker='unknown')
+
     def test_instance_get_active_by_window_joined(self):
         now = datetime.datetime(2013, 10, 10, 17, 16, 37, 156701)
         start_time = now - datetime.timedelta(minutes=10)

nova/tests/unit/objects/test_instance.py

@@ -1757,7 +1757,8 @@ class _TestInstanceListObject(object):
 
         def fake_instance_get_active_by_window_joined(context, begin, end,
                                                       project_id, host,
-                                                      columns_to_join):
+                                                      columns_to_join,
+                                                      limit=None, marker=None):
             # make sure begin is tz-aware
             self.assertIsNotNone(begin.utcoffset())
             self.assertIsNone(end)

nova/tests/unit/objects/test_objects.py

@@ -1108,7 +1108,7 @@ object_data = {
     'InstanceGroup': '1.10-1a0c8c7447dc7ecb9da53849430c4a5f',
     'InstanceGroupList': '1.7-be18078220513316abd0ae1b2d916873',
     'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e',
-    'InstanceList': '2.1-e64b9f623db6370b22ec910461f06a52',
+    'InstanceList': '2.2-ff71772c7bf6d72f6ef6eee0199fb1c9',
     'InstanceMapping': '1.0-65de80c491f54d19374703c0753c4d47',
     'InstanceMappingList': '1.0-9e982e3de1613b9ada85e35f69b23d47',
     'InstanceNUMACell': '1.3-6991a20992c5faa57fae71a45b40241b',

releasenotes/notes/pagination-for-usage-a313397f9a7e9a70.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Added microversion v2.40 which introduces pagination support for usage
+    with the help of new optional parameters 'limit' and 'marker'. If 'limit'
+    isn't provided, it will default to the configurable max limit which is
+    currently 1000.
+
+    ::
+
+        /os-simple-tenant-usage?limit={limit}&marker={instance_uuid}
+        /os-simple-tenant-usage/{tenant}?limit={limit}&marker={instance_uuid}
+
+    Older microversions will not accept these new paging query parameters,
+    but they will start to silently limit by the max limit to encourage the
+    adoption of this new microversion, and circumvent the existing possibility
+    DoS-like usage requests on systems with thousands of instances.

tests-py3.txt

@@ -5,7 +5,6 @@ nova.tests.unit.api.openstack.compute.test_security_group_default_rules.TestSecu
 nova.tests.unit.api.openstack.compute.test_security_group_default_rules.TestSecurityGroupDefaultRulesV21
 nova.tests.unit.api.openstack.compute.test_security_groups.SecurityGroupsOutputTestV21
 nova.tests.unit.api.openstack.compute.test_security_groups.TestSecurityGroupRulesV21
-nova.tests.unit.api.openstack.compute.test_simple_tenant_usage.SimpleTenantUsageTestV21
 nova.tests.unit.api.openstack.compute.test_user_data.ServersControllerCreateTest
 nova.tests.unit.compute.test_compute.ComputeAPITestCase.test_create_with_base64_user_data
 nova.tests.unit.compute.test_compute_cells.CellsComputeAPITestCase.test_create_with_base64_user_data