openstack/nova
Code Issues Proposed changes

Merge "Simple tenant usage pagination"

Browse Source
This commit is contained in:
Jenkins 2016-12-15 18:25:19 +00:00 committed by Gerrit Code Review
commit bcc1768042
30 changed files with 608 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api-ref/source/os-simple-tenant-usage.inc
View File

@ -27,6 +27,8 @@ Request
- detailed: detailed_simple_tenant_usage
- end: end_simple_tenant_usage
- start: start_simple_tenant_usage
- limit: usage_limit
- marker: usage_marker
Response
--------
@ -60,20 +62,20 @@ Response
If the ``detailed`` query parameter is not specified or
is set to other than 1 (e.g. ``detailed=0``), the response is as follows:
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get.json
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json
:language: javascript
If the ``detailed`` query parameter is set to one (``detailed=1``),
the response includes ``server_usages`` information for each tenant.
The response is as follows:
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get-detail.json
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json
:language: javascript
Show Usage Statistics For Tenant
================================
.. rest_method:: GET /os-simple-tenant-usage/{tenant_id}
.. rest_method:: GET /os-simple-tenant-usage/v2.40/{tenant_id}
Shows usage statistics for a tenant.
@ -89,6 +91,8 @@ Request
- tenant_id: tenant_id
- end: end_simple_tenant_usage
- start: start_simple_tenant_usage
- limit: usage_limit
- marker: usage_marker
Response
--------
@ -119,5 +123,5 @@ Response
**Example Show Usage Details For Tenant: JSON response**
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/simple-tenant-usage-get-specific.json
.. literalinclude:: ../../doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json
:language: javascript

19
api-ref/source/parameters.yaml
View File

@ -717,6 +717,25 @@ tags_query:
all tags in this list will be returned. Boolean expression in this
case is 't1 AND t2'. Tags in query must be separated by comma.
min_version: 2.26
usage_limit:
description: |
Requests a page size of items. Calculate usage for the limited number of
instances. Use the ``limit`` parameter to make an initial limited request
and use the last-seen instance UUID from the response as the ``marker``
parameter value in a subsequent limited request.
in: query
required: false
type: integer
min_version: 2.40
usage_marker:
description: |
The last-seen item. Use the ``limit`` parameter to make an initial limited
request and use the last-seen instance UUID from the response as the
``marker`` parameter value in a subsequent limited request.
in: query
required: false
type: string
min_version: 2.40
user_id_query_quota:
description: |
ID of user to list the quotas for.

2
doc/api_samples/keypairs/v2.35/keypairs-list-user2-resp.json
View File

@ -11,7 +11,7 @@
],
"keypairs_links": [
{
"href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/keypairs?user_id=user2&limit=1&marker=keypair-5d935425-31d5-48a7-a0f1-e76e9813f2c3",
"href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/keypairs?limit=1&marker=keypair-5d935425-31d5-48a7-a0f1-e76e9813f2c3&user_id=user2",
"rel": "next"
}
]

35
doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json Normal file
View File

@ -0,0 +1,35 @@
{
"tenant_usages": [
{
"start": "2012-10-08T20:10:44.587336",
"stop": "2012-10-08T21:10:44.587336",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0,
"server_usages": [
{
"ended_at": null,
"flavor": "m1.tiny",
"hours": 1.0,
"instance_id": "1f1deceb-17b5-4c04-84c7-e0d4499c8fe0",
"local_gb": 1,
"memory_mb": 512,
"name": "instance-2",
"started_at": "2012-10-08T20:10:44.541277",
"state": "active",
"tenant_id": "6f70656e737461636b20342065766572",
"uptime": 3600,
"vcpus": 1
}
]
}
],
"tenant_usages_links": [
{
"href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage?detailed=1&end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
"rel": "next"
}
]
}

33
doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json Normal file
View File

@ -0,0 +1,33 @@
{
"tenant_usage": {
"server_usages": [
{
"ended_at": null,
"flavor": "m1.tiny",
"hours": 1.0,
"instance_id": "1f1deceb-17b5-4c04-84c7-e0d4499c8fe0",
"local_gb": 1,
"memory_mb": 512,
"name": "instance-2",
"started_at": "2012-10-08T20:10:44.541277",
"state": "active",
"tenant_id": "6f70656e737461636b20342065766572",
"uptime": 3600,
"vcpus": 1
}
],
"start": "2012-10-08T20:10:44.587336",
"stop": "2012-10-08T21:10:44.587336",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0
},
"tenant_usage_links": [
{
"href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage/6f70656e737461636b20342065766572?end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
"rel": "next"
}
]
}

19
doc/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json Normal file
View File

@ -0,0 +1,19 @@
{
"tenant_usages": [
{
"start": "2012-10-08T21:10:44.587336",
"stop": "2012-10-08T22:10:44.587336",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0
}
],
"tenant_usages_links": [
{
"href": "http://openstack.example.com/v2.1/6f70656e737461636b20342065766572/os-simple-tenant-usage?end=2016-10-12+18%3A22%3A04.868106&limit=1&marker=1f1deceb-17b5-4c04-84c7-e0d4499c8fe0&start=2016-10-12+18%3A22%3A04.868106",
"rel": "next"
}
]
}

2
doc/api_samples/versions/v21-version-get-resp.json
View File

@ -19,7 +19,7 @@
}
],
"status": "CURRENT",
"version": "2.39",
"version": "2.40",
"min_version": "2.1",
"updated": "2013-07-23T11:33:21Z"
}

2
doc/api_samples/versions/versions-get-resp.json
View File

@ -22,7 +22,7 @@
}
],
"status": "CURRENT",
"version": "2.39",
"version": "2.40",
"min_version": "2.1",
"updated": "2013-07-23T11:33:21Z"
}

3
nova/api/openstack/api_version_request.py
View File

@ -96,6 +96,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
* 2.38 - Add a condition to return HTTPBadRequest if invalid status is
provided for listing servers.
* 2.39 - Deprecates image-metadata proxy API
* 2.40 - Adds simple tenant usage pagination support.
"""
# The minimum and maximum versions of the API supported
@ -104,7 +105,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
# Note(cyeoh): This only applies for the v2.1 API once microversions
# support is fully merged. It does not affect the V2 API.
_MIN_API_VERSION = "2.1"
_MAX_API_VERSION = "2.39"
_MAX_API_VERSION = "2.40"
DEFAULT_API_VERSION = _MIN_API_VERSION
# Almost all proxy APIs which related to network, images and baremetal

2
nova/api/openstack/common.py
View File

@ -401,7 +401,7 @@ class ViewBuilder(object):
def _get_next_link(self, request, identifier, collection_name):
"""Return href string with proper limit and marker params."""
params = request.params.copy()
params = collections.OrderedDict(sorted(request.params.items()))
params["marker"] = identifier
prefix = self._update_compute_link_prefix(request.application_url)
url = url_join(prefix,

99
nova/api/openstack/compute/simple_tenant_usage.py
View File

@ -21,13 +21,17 @@ import six
import six.moves.urllib.parse as urlparse
from webob import exc
from nova.api.openstack import common
from nova.api.openstack.compute.views import usages as usages_view
from nova.api.openstack import extensions
from nova.api.openstack import wsgi
import nova.conf
from nova import exception
from nova.i18n import _
from nova import objects
from nova.policies import simple_tenant_usage as stu_policies
CONF = nova.conf.CONF
ALIAS = "os-simple-tenant-usage"
@ -39,6 +43,9 @@ def parse_strtime(dstr, fmt):
class SimpleTenantUsageController(wsgi.Controller):
_view_builder_class = usages_view.ViewBuilder
def _hours_for(self, instance, period_start, period_stop):
launched_at = instance.launched_at
terminated_at = instance.terminated_at
@ -97,14 +104,16 @@ class SimpleTenantUsageController(wsgi.Controller):
return flavor_ref
def _tenant_usages_for_period(self, context, period_start,
period_stop, tenant_id=None, detailed=True):
def _tenant_usages_for_period(self, context, period_start, period_stop,
tenant_id=None, detailed=True, limit=None,
marker=None):
instances = objects.InstanceList.get_active_by_window_joined(
context, period_start, period_stop, tenant_id,
expected_attrs=['flavor'])
expected_attrs=['flavor'], limit=limit, marker=marker)
rval = {}
flavors = {}
all_server_usages = []
for instance in instances:
info = {}
@ -170,10 +179,11 @@ class SimpleTenantUsageController(wsgi.Controller):
info['hours'])
summary['total_hours'] += info['hours']
all_server_usages.append(info)
if detailed:
summary['server_usages'].append(info)
return rval.values()
return list(rval.values()), all_server_usages
def _parse_datetime(self, dtstr):
if not dtstr:
@ -216,9 +226,31 @@ class SimpleTenantUsageController(wsgi.Controller):
detailed = env.get('detailed', ['0'])[0] == '1'
return (period_start, period_stop, detailed)
@wsgi.Controller.api_version("2.40")
@extensions.expected_errors(400)
def index(self, req):
"""Retrieve tenant_usage for all tenants."""
return self._index(req, links=True)
@wsgi.Controller.api_version("2.1", "2.39") # noqa
@extensions.expected_errors(400)
def index(self, req):
"""Retrieve tenant_usage for all tenants."""
return self._index(req)
@wsgi.Controller.api_version("2.40")
@extensions.expected_errors(400)
def show(self, req, id):
"""Retrieve tenant_usage for a specified tenant."""
return self._show(req, id, links=True)
@wsgi.Controller.api_version("2.1", "2.39") # noqa
@extensions.expected_errors(400)
def show(self, req, id):
"""Retrieve tenant_usage for a specified tenant."""
return self._show(req, id)
def _index(self, req, links=False):
context = req.environ['nova.context']
context.can(stu_policies.POLICY_ROOT % 'list')
@ -232,15 +264,29 @@ class SimpleTenantUsageController(wsgi.Controller):
now = timeutils.parse_isotime(timeutils.utcnow().isoformat())
if period_stop > now:
period_stop = now
usages = self._tenant_usages_for_period(context,
period_start,
period_stop,
detailed=detailed)
return {'tenant_usages': usages}
@extensions.expected_errors(400)
def show(self, req, id):
"""Retrieve tenant_usage for a specified tenant."""
marker = None
limit = CONF.api.max_limit
if links:
limit, marker = common.get_limit_and_marker(req)
try:
usages, server_usages = self._tenant_usages_for_period(
context, period_start, period_stop, detailed=detailed,
limit=limit, marker=marker)
except exception.MarkerNotFound as e:
raise exc.HTTPBadRequest(explanation=e.format_message())
tenant_usages = {'tenant_usages': usages}
if links:
usages_links = self._view_builder.get_links(req, server_usages)
if usages_links:
tenant_usages['tenant_usages_links'] = usages_links
return tenant_usages
def _show(self, req, id, links=False):
tenant_id = id
context = req.environ['nova.context']
@ -256,16 +302,33 @@ class SimpleTenantUsageController(wsgi.Controller):
now = timeutils.parse_isotime(timeutils.utcnow().isoformat())
if period_stop > now:
period_stop = now
usage = self._tenant_usages_for_period(context,
period_start,
period_stop,
tenant_id=tenant_id,
detailed=True)
marker = None
limit = CONF.api.max_limit
if links:
limit, marker = common.get_limit_and_marker(req)
try:
usage, server_usages = self._tenant_usages_for_period(
context, period_start, period_stop, tenant_id=tenant_id,
detailed=True, limit=limit, marker=marker)
except exception.MarkerNotFound as e:
raise exc.HTTPBadRequest(explanation=e.format_message())
if len(usage):
usage = list(usage)[0]
else:
usage = {}
return {'tenant_usage': usage}
tenant_usage = {'tenant_usage': usage}
if links:
usages_links = self._view_builder.get_links(
req, server_usages, tenant_id=tenant_id)
if usages_links:
tenant_usage['tenant_usage_links'] = usages_links
return tenant_usage
class SimpleTenantUsage(extensions.V21APIExtensionBase):

28
nova/api/openstack/compute/views/usages.py Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2016 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nova.api.openstack import common
class ViewBuilder(common.ViewBuilder):
_collection_name = "os-simple-tenant-usage"
def get_links(self, request, server_usages, tenant_id=None):
coll_name = self._collection_name
if tenant_id:
coll_name = self._collection_name + '/{}'.format(tenant_id)
return self._get_collection_links(
request, server_usages, coll_name, 'instance_id')

19
nova/api/openstack/rest_api_version_history.rst
View File

@ -426,3 +426,22 @@ user documentation.
option `image_property_quota` should be used to control the quota of
image metadatas. Also, removes the `maxImageMeta` field from `os-limits`
API response.
2.40
----
Optional parameters 'limit' and 'marker' were added to the GET
/os-simple-tenant-usage and GET os-simple-tenant-usage/{tenant_id}
requests. The aggregate usage data no longer reflects all instances for a
tenant, but rather just the paginated instances ordered by instance id ASC.
API consumers will need to stitch the aggregate data back up (add the totals)
if a tenant's instances span several pages.
GET /os-simple-tenant-usage?limit={limit}&marker={instance_uuid}
GET /os-simple-tenant-usage/{tenant_id}?limit={limit}&marker={instance_uuid}
Older versions of the `os-simple-tenant-usage` endpoints will not accept
these new paging query parameters, but they will start to silently limit by
`CONF.api.max_limit` to encourage the adoption of this new microversion,
and circumvent the existing possibility DoS-like usage requests on systems
with thousands of instances.

6
nova/db/api.py
View File

@ -765,7 +765,8 @@ def instance_get_all_by_filters_sort(context, filters, limit=None,
def instance_get_active_by_window_joined(context, begin, end=None,
project_id=None, host=None,
columns_to_join=None):
columns_to_join=None, limit=None,
marker=None):
"""Get instances and joins active during a certain time window.
Specifying a project_id will filter for a certain project.
@ -773,7 +774,8 @@ def instance_get_active_by_window_joined(context, begin, end=None,
"""
return IMPL.instance_get_active_by_window_joined(context, begin, end,
project_id, host,
columns_to_join=columns_to_join)
columns_to_join=columns_to_join,
limit=limit, marker=marker)
def instance_get_all_by_host(context, host, columns_to_join=None):

13
nova/db/sqlalchemy/api.py
View File

@ -2513,7 +2513,8 @@ def process_sort_params(sort_keys, sort_dirs,
@pick_context_manager_reader_allow_async
def instance_get_active_by_window_joined(context, begin, end=None,
project_id=None, host=None,
columns_to_join=None):
columns_to_join=None, limit=None,
marker=None):
"""Return instances and joins that were active during window."""
query = context.session.query(models.Instance)
@ -2539,6 +2540,16 @@ def instance_get_active_by_window_joined(context, begin, end=None,
if host:
query = query.filter_by(host=host)
if marker is not None:
try:
marker = _instance_get_by_uuid(
context.elevated(read_deleted='yes'), marker)
except exception.InstanceNotFound:
raise exception.MarkerNotFound(marker=marker)
query = sqlalchemyutils.paginate_query(
query, models.Instance, limit, ['project_id', 'uuid'], marker=marker)
return _instances_fill_metadata(context, query.all(), manual_joins)

22
nova/objects/instance.py
View File

@ -1184,7 +1184,8 @@ def _make_instance_list(context, inst_list, db_inst_list, expected_attrs):
class InstanceList(base.ObjectListBase, base.NovaObject):
# Version 2.0: Initial Version
# Version 2.1: Add get_uuids_by_host()
VERSION = '2.1'
# Version 2.2: Pagination for get_active_by_window_joined()
VERSION = '2.2'
fields = {
'objects': fields.ListOfObjectsField('Instance'),
@ -1269,16 +1270,16 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
@db.select_db_reader_mode
def _db_instance_get_active_by_window_joined(
context, begin, end, project_id, host, columns_to_join,
use_slave=False):
use_slave=False, limit=None, marker=None):
return db.instance_get_active_by_window_joined(
context, begin, end, project_id, host,
columns_to_join=columns_to_join)
columns_to_join=columns_to_join, limit=limit, marker=marker)
@base.remotable_classmethod
def _get_active_by_window_joined(cls, context, begin, end=None,
project_id=None, host=None,
expected_attrs=None,
use_slave=False):
expected_attrs=None, use_slave=False,
limit=None, marker=None):
# NOTE(mriedem): We need to convert the begin/end timestamp strings
# to timezone-aware datetime objects for the DB API call.
begin = timeutils.parse_isotime(begin)
@ -1286,15 +1287,15 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
db_inst_list = cls._db_instance_get_active_by_window_joined(
context, begin, end, project_id, host,
columns_to_join=_expected_cols(expected_attrs),
use_slave=use_slave)
use_slave=use_slave, limit=limit, marker=marker)
return _make_instance_list(context, cls(), db_inst_list,
expected_attrs)
@classmethod
def get_active_by_window_joined(cls, context, begin, end=None,
project_id=None, host=None,
expected_attrs=None,
use_slave=False):
expected_attrs=None, use_slave=False,
limit=None, marker=None):
"""Get instances and joins active during a certain time window.
:param:context: nova request context
@ -1305,6 +1306,8 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
:param:expected_attrs: list of related fields that can be joined
in the database layer when querying for instances
:param use_slave if True, ship this query off to a DB slave
:param limit: maximum number of instances to return per page
:param marker: last instance uuid from the previous page
:returns: InstanceList
"""
@ -1315,7 +1318,8 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
return cls._get_active_by_window_joined(context, begin, end,
project_id, host,
expected_attrs,
use_slave=use_slave)
use_slave=use_slave,
limit=limit, marker=marker)
@base.remotable_classmethod
def get_by_security_group_id(cls, context, security_group_id):

2
nova/tests/functional/api_sample_tests/api_samples/keypairs/v2.35/keypairs-list-user2-resp.json.tpl
View File

@ -11,7 +11,7 @@
],
"keypairs_links": [
{
"href": "%(versioned_compute_endpoint)s/keypairs?user_id=user2&limit=1&marker=%(keypair_name)s",
"href": "%(versioned_compute_endpoint)s/keypairs?limit=1&marker=%(keypair_name)s&user_id=user2",
"rel": "next"
}
]

35
nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-detail.json.tpl Normal file
View File

@ -0,0 +1,35 @@
{
"tenant_usages": [
{
"start": "%(strtime)s",
"stop": "%(strtime)s",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0,
"server_usages": [
{
"ended_at": null,
"flavor": "m1.tiny",
"hours": 1.0,
"instance_id": "%(uuid)s",
"local_gb": 1,
"memory_mb": 512,
"name": "instance-2",
"started_at": "%(strtime)s",
"state": "active",
"tenant_id": "6f70656e737461636b20342065766572",
"uptime": 3600,
"vcpus": 1
}
]
}
],
"tenant_usages_links": [
{
"href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage?detailed=1&end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
"rel": "next"
}
]
}

33
nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get-specific.json.tpl Normal file
View File

@ -0,0 +1,33 @@
{
"tenant_usage": {
"server_usages": [
{
"ended_at": null,
"flavor": "m1.tiny",
"hours": 1.0,
"instance_id": "%(uuid)s",
"local_gb": 1,
"memory_mb": 512,
"name": "instance-2",
"started_at": "%(strtime)s",
"state": "active",
"tenant_id": "6f70656e737461636b20342065766572",
"uptime": 3600,
"vcpus": 1
}
],
"start": "%(strtime)s",
"stop": "%(strtime)s",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0
},
"tenant_usage_links": [
{
"href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage/%(tenant_id)s?end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
"rel": "next"
}
]
}

19
nova/tests/functional/api_sample_tests/api_samples/os-simple-tenant-usage/v2.40/simple-tenant-usage-get.json.tpl Normal file
View File

@ -0,0 +1,19 @@
{
"tenant_usages": [
{
"start": "%(strtime)s",
"stop": "%(strtime)s",
"tenant_id": "6f70656e737461636b20342065766572",
"total_hours": 1.0,
"total_local_gb_usage": 1.0,
"total_memory_mb_usage": 512.0,
"total_vcpus_usage": 1.0
}
],
"tenant_usages_links": [
{
"href": "%(versioned_compute_endpoint)s/os-simple-tenant-usage?end=%(strtime_url)s&limit=1&marker=%(uuid)s&start=%(strtime_url)s",
"rel": "next"
}
]
}

2
nova/tests/functional/api_sample_tests/api_samples/servers/server-create-req-v237.json.tpl
View File

@ -2,7 +2,7 @@
"server" : {
"accessIPv4": "%(access_ip_v4)s",
"accessIPv6": "%(access_ip_v6)s",
"name" : "new-server-test",
"name" : "%(name)s",
"imageRef" : "%(image_id)s",
"flavorRef" : "1",
"availability_zone": "nova",

3
nova/tests/functional/api_sample_tests/test_servers.py
View File

@ -46,7 +46,7 @@ class ServersSampleBase(api_sample_base.ApiSampleTestBaseV21):
avr.APIVersionRequest(min), avr.APIVersionRequest(max)):
return name
def _post_server(self, use_common_server_api_samples=True):
def _post_server(self, use_common_server_api_samples=True, name=None):
# param use_common_server_api_samples: Boolean to set whether tests use
# common sample files for server post request and response.
# Default is True which means _get_sample_path method will fetch the
@ -63,6 +63,7 @@ class ServersSampleBase(api_sample_base.ApiSampleTestBaseV21):
'user_data': self.user_data,
'uuid': '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}'
'-[0-9a-f]{4}-[0-9a-f]{12}',
'name': 'new-server-test' if name is None else name,
}
orig_value = self.__class__._use_common_server_api_samples

57
nova/tests/functional/api_sample_tests/test_simple_tenant_usage.py
View File

@ -15,6 +15,7 @@
import datetime
import urllib
import mock
from oslo_utils import timeutils
from nova.tests.functional.api_sample_tests import test_servers
@ -67,3 +68,59 @@ class SimpleTenantUsageSampleJsonTest(test_servers.ServersSampleBase):
urllib.urlencode(self.query)))
self._verify_response('simple-tenant-usage-get-specific', {},
response, 200)
class SimpleTenantUsageV240Test(test_servers.ServersSampleBase):
sample_dir = 'os-simple-tenant-usage'
microversion = '2.40'
scenarios = [('v2_40', {'api_major_version': 'v2.1'})]
def setUp(self):
super(SimpleTenantUsageV240Test, self).setUp()
self.api.microversion = self.microversion
started = timeutils.utcnow()
now = started + datetime.timedelta(hours=1)
timeutils.set_time_override(started)
with mock.patch('oslo_utils.uuidutils.generate_uuid') as mock_uuids:
# make uuids incrementing, so that sort order is deterministic
uuid_format = '1f1deceb-17b5-4c04-84c7-e0d4499c8f%02d'
mock_uuids.side_effect = [uuid_format % x for x in range(100)]
self.instance1_uuid = self._post_server(name='instance-1')
self.instance2_uuid = self._post_server(name='instance-2')
self.instance3_uuid = self._post_server(name='instance-3')
timeutils.set_time_override(now)
self.query = {
'start': str(started),
'end': str(now),
'limit': '1',
'marker': self.instance1_uuid,
}
def tearDown(self):
super(SimpleTenantUsageV240Test, self).tearDown()
timeutils.clear_time_override()
def test_get_tenants_usage(self):
url = 'os-simple-tenant-usage?%s'
response = self._do_get(url % (urllib.urlencode(self.query)))
template_name = 'simple-tenant-usage-get'
self._verify_response(template_name, {}, response, 200)
def test_get_tenants_usage_with_detail(self):
query = self.query.copy()
query.update({'detailed': 1})
url = 'os-simple-tenant-usage?%s'
response = self._do_get(url % (urllib.urlencode(query)))
template_name = 'simple-tenant-usage-get-detail'
self._verify_response(template_name, {}, response, 200)
def test_get_tenant_usage_details(self):
tenant_id = astb.PROJECT_ID
url = 'os-simple-tenant-usage/{tenant}?%s'.format(tenant=tenant_id)
response = self._do_get(url % (urllib.urlencode(self.query)))
template_name = 'simple-tenant-usage-get-specific'
subs = {'tenant_id': self.api.project_id}
self._verify_response(template_name, subs, response, 200)

3
nova/tests/functional/api_samples_test_base.py
View File

@ -410,6 +410,8 @@ class ApiSampleTestBase(integrated_helpers._IntegratedTestBase):
text = r'(\\"|[^"])*'
isotime_re = '\d{4}-[0,1]\d-[0-3]\dT\d{2}:\d{2}:\d{2}Z'
strtime_re = '\d{4}-[0,1]\d-[0-3]\dT\d{2}:\d{2}:\d{2}\.\d{6}'
strtime_url_re = ('\d{4}-[0,1]\d-[0-3]\d'
'\+\d{2}\%3A\d{2}\%3A\d{2}\.\d{6}')
xmltime_re = ('\d{4}-[0,1]\d-[0-3]\d '
'\d{2}:\d{2}:\d{2}'
'(\.\d{6})?(\+00:00)?')
@ -419,6 +421,7 @@ class ApiSampleTestBase(integrated_helpers._IntegratedTestBase):
return {
'isotime': isotime_re,
'strtime': strtime_re,
'strtime_url': strtime_url_re,
'strtime_or_none': r'None|%s' % strtime_re,
'xmltime': xmltime_re,
'password': '[0-9a-zA-Z]{1,12}',

141
nova/tests/unit/api/openstack/compute/test_simple_tenant_usage.py
View File

@ -24,6 +24,7 @@ import webob
from nova.api.openstack.compute import simple_tenant_usage as \
simple_tenant_usage_v21
from nova.compute import vm_states
import nova.conf
from nova import context
from nova import exception
from nova import objects
@ -32,6 +33,10 @@ from nova import test
from nova.tests.unit.api.openstack import fakes
from nova.tests import uuidsentinel as uuids
CONF = nova.conf.CONF
SERVERS = 5
TENANTS = 2
HOURS = 24
@ -88,16 +93,18 @@ def _fake_instance(start, end, instance_id, tenant_id,
@classmethod
def fake_get_active_by_window_joined(cls, context, begin, end=None,
project_id=None, host=None,
expected_attrs=None, use_slave=False):
expected_attrs=None, use_slave=False,
limit=None, marker=None):
return objects.InstanceList(objects=[
_fake_instance(START, STOP, x,
project_id or 'faketenant_%s' % (x / SERVERS))
project_id or 'faketenant_%s' % (x // SERVERS))
for x in range(TENANTS * SERVERS)])
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined',
fake_get_active_by_window_joined)
class SimpleTenantUsageTestV21(test.TestCase):
version = '2.1'
policy_rule_prefix = "os_compute_api:os-simple-tenant-usage"
controller = simple_tenant_usage_v21.SimpleTenantUsageController()
@ -113,11 +120,16 @@ class SimpleTenantUsageTestV21(test.TestCase):
'faketenant_1',
is_admin=False)
def _test_verify_index(self, start, stop):
req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
(start.isoformat(), stop.isoformat()))
def _test_verify_index(self, start, stop, limit=None):
url = '?start=%s&end=%s'
if limit:
url += '&limit=%s' % (limit)
req = fakes.HTTPRequest.blank(url %
(start.isoformat(), stop.isoformat()),
version=self.version)
req.environ['nova.context'] = self.admin_context
res_dict = self.controller.index(req)
usages = res_dict['tenant_usages']
for i in range(TENANTS):
self.assertEqual(SERVERS * HOURS, int(usages[i]['total_hours']))
@ -129,6 +141,12 @@ class SimpleTenantUsageTestV21(test.TestCase):
int(usages[i]['total_vcpus_usage']))
self.assertFalse(usages[i].get('server_usages'))
if limit:
self.assertIn('tenant_usages_links', res_dict)
self.assertEqual('next', res_dict['tenant_usages_links'][0]['rel'])
else:
self.assertNotIn('tenant_usages_links', res_dict)
def test_verify_index(self):
self._test_verify_index(START, STOP)
@ -145,7 +163,8 @@ class SimpleTenantUsageTestV21(test.TestCase):
def _get_tenant_usages(self, detailed=''):
req = fakes.HTTPRequest.blank('?detailed=%s&start=%s&end=%s' %
(detailed, START.isoformat(), STOP.isoformat()))
(detailed, START.isoformat(), STOP.isoformat()),
version=self.version)
req.environ['nova.context'] = self.admin_context
# Make sure that get_active_by_window_joined is only called with
@ -155,8 +174,8 @@ class SimpleTenantUsageTestV21(test.TestCase):
def fake_get_active_by_window_joined(context, begin, end=None,
project_id=None, host=None,
expected_attrs=None,
use_slave=False):
expected_attrs=None, use_slave=False,
limit=None, marker=None):
self.assertEqual(['flavor'], expected_attrs)
return orig_get_active_by_window_joined(context, begin, end,
project_id, host,
@ -186,12 +205,15 @@ class SimpleTenantUsageTestV21(test.TestCase):
for i in range(TENANTS):
self.assertIsNone(usages[i].get('server_usages'))
def _test_verify_show(self, start, stop):
def _test_verify_show(self, start, stop, limit=None):
tenant_id = 1
req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
(start.isoformat(), stop.isoformat()))
url = '?start=%s&end=%s'
if limit:
url += '&limit=%s' % (limit)
req = fakes.HTTPRequest.blank(url %
(start.isoformat(), stop.isoformat()),
version=self.version)
req.environ['nova.context'] = self.user_context
res_dict = self.controller.show(req, tenant_id)
usage = res_dict['tenant_usage']
@ -207,9 +229,16 @@ class SimpleTenantUsageTestV21(test.TestCase):
self.assertEqual(HOURS, int(servers[j]['hours']))
self.assertIn(servers[j]['instance_id'], server_uuids)
if limit:
self.assertIn('tenant_usage_links', res_dict)
self.assertEqual('next', res_dict['tenant_usage_links'][0]['rel'])
else:
self.assertNotIn('tenant_usage_links', res_dict)
def test_verify_show_cannot_view_other_tenant(self):
req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
(START.isoformat(), STOP.isoformat()))
(START.isoformat(), STOP.isoformat()),
version=self.version)
req.environ['nova.context'] = self.alt_user_context
rules = {
@ -227,20 +256,23 @@ class SimpleTenantUsageTestV21(test.TestCase):
def test_get_tenants_usage_with_bad_start_date(self):
future = NOW + datetime.timedelta(hours=HOURS)
req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
(future.isoformat(), NOW.isoformat()))
(future.isoformat(), NOW.isoformat()),
version=self.version)
req.environ['nova.context'] = self.user_context
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.show, req, 'faketenant_0')
def test_get_tenants_usage_with_invalid_start_date(self):
req = fakes.HTTPRequest.blank('?start=%s&end=%s' %
("xxxx", NOW.isoformat()))
("xxxx", NOW.isoformat()),
version=self.version)
req.environ['nova.context'] = self.user_context
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.show, req, 'faketenant_0')
def _test_get_tenants_usage_with_one_date(self, date_url_param):
req = fakes.HTTPRequest.blank('?%s' % date_url_param)
req = fakes.HTTPRequest.blank('?%s' % date_url_param,
version=self.version)
req.environ['nova.context'] = self.user_context
res = self.controller.show(req, 'faketenant_0')
self.assertIn('tenant_usage', res)
@ -254,6 +286,83 @@ class SimpleTenantUsageTestV21(test.TestCase):
'start=%s' % (NOW - datetime.timedelta(5)).isoformat())
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined',
fake_get_active_by_window_joined)
class SimpleTenantUsageTestV40(SimpleTenantUsageTestV21):
version = '2.40'
def test_next_links_show(self):
self._test_verify_show(START, STOP, limit=SERVERS * TENANTS)
def test_next_links_index(self):
self._test_verify_index(START, STOP, limit=SERVERS * TENANTS)
class SimpleTenantUsageLimitsTestV21(test.TestCase):
version = '2.1'
def setUp(self):
super(SimpleTenantUsageLimitsTestV21, self).setUp()
self.controller = simple_tenant_usage_v21.SimpleTenantUsageController()
self.tenant_id = 1
def _get_request(self, url):
url = url % (START.isoformat(), STOP.isoformat())
return fakes.HTTPRequest.blank(url, version=self.version)
def assert_limit(self, mock_get, limit):
mock_get.assert_called_once_with(
mock.ANY, mock.ANY, mock.ANY, mock.ANY, expected_attrs=['flavor'],
limit=1000, marker=None)
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_limit_defaults_to_conf_max_limit_show(self, mock_get):
req = self._get_request('?start=%s&end=%s')
self.controller.show(req, self.tenant_id)
self.assert_limit(mock_get, CONF.api.max_limit)
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_limit_defaults_to_conf_max_limit_index(self, mock_get):
req = self._get_request('?start=%s&end=%s')
self.controller.index(req)
self.assert_limit(mock_get, CONF.api.max_limit)
class SimpleTenantUsageLimitsTestV240(SimpleTenantUsageLimitsTestV21):
version = '2.40'
def assert_limit_and_marker(self, mock_get, limit, marker):
mock_get.assert_called_once_with(
mock.ANY, mock.ANY, mock.ANY, mock.ANY, expected_attrs=['flavor'],
limit=3, marker=marker)
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_limit_and_marker_show(self, mock_get):
req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
self.controller.show(req, self.tenant_id)
self.assert_limit_and_marker(mock_get, 3, 'some-marker')
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_limit_and_marker_index(self, mock_get):
req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
self.controller.index(req)
self.assert_limit_and_marker(mock_get, 3, 'some-marker')
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_marker_not_found_show(self, mock_get):
mock_get.side_effect = exception.MarkerNotFound(marker='some-marker')
req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
self.assertRaises(
webob.exc.HTTPBadRequest, self.controller.show, req, 1)
@mock.patch('nova.objects.InstanceList.get_active_by_window_joined')
def test_marker_not_found_index(self, mock_get):
mock_get.side_effect = exception.MarkerNotFound(marker='some-marker')
req = self._get_request('?start=%s&end=%s&limit=3&marker=some-marker')
self.assertRaises(
webob.exc.HTTPBadRequest, self.controller.index, req)
class SimpleTenantUsageControllerTestV21(test.TestCase):
controller = simple_tenant_usage_v21.SimpleTenantUsageController()

35
nova/tests/unit/db/test_db_api.py
View File

@ -1199,6 +1199,41 @@ class SqlAlchemyDbApiTestCase(DbTestCase):
self.assertEqual(2, len(result))
self.assertEqual(six.text_type, type(result[0]))
@mock.patch('oslo_utils.uuidutils.generate_uuid')
def test_instance_get_active_by_window_joined_paging(self, mock_uuids):
mock_uuids.side_effect = ['BBB', 'ZZZ', 'AAA', 'CCC']
ctxt = context.get_admin_context()
now = datetime.datetime(2015, 10, 2)
self.create_instance_with_args(project_id='project-ZZZ')
self.create_instance_with_args(project_id='project-ZZZ')
self.create_instance_with_args(project_id='project-ZZZ')
self.create_instance_with_args(project_id='project-AAA')
# no limit or marker
result = sqlalchemy_api.instance_get_active_by_window_joined(
ctxt, begin=now, columns_to_join=[])
actual_uuids = [row['uuid'] for row in result]
self.assertEqual(['CCC', 'AAA', 'BBB', 'ZZZ'], actual_uuids)
# just limit
result = sqlalchemy_api.instance_get_active_by_window_joined(
ctxt, begin=now, columns_to_join=[], limit=2)
actual_uuids = [row['uuid'] for row in result]
self.assertEqual(['CCC', 'AAA'], actual_uuids)
# limit & marker
result = sqlalchemy_api.instance_get_active_by_window_joined(
ctxt, begin=now, columns_to_join=[], limit=2, marker='CCC')
actual_uuids = [row['uuid'] for row in result]
self.assertEqual(['AAA', 'BBB'], actual_uuids)
# unknown marker
self.assertRaises(
exception.MarkerNotFound,
sqlalchemy_api.instance_get_active_by_window_joined,
ctxt, begin=now, columns_to_join=[], limit=2, marker='unknown')
def test_instance_get_active_by_window_joined(self):
now = datetime.datetime(2013, 10, 10, 17, 16, 37, 156701)
start_time = now - datetime.timedelta(minutes=10)

3
nova/tests/unit/objects/test_instance.py
View File

@ -1757,7 +1757,8 @@ class _TestInstanceListObject(object):
def fake_instance_get_active_by_window_joined(context, begin, end,
project_id, host,
columns_to_join):
columns_to_join,
limit=None, marker=None):
# make sure begin is tz-aware
self.assertIsNotNone(begin.utcoffset())
self.assertIsNone(end)

2
nova/tests/unit/objects/test_objects.py
View File

@ -1108,7 +1108,7 @@ object_data = {
'InstanceGroup': '1.10-1a0c8c7447dc7ecb9da53849430c4a5f',
'InstanceGroupList': '1.7-be18078220513316abd0ae1b2d916873',
'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e',
'InstanceList': '2.1-e64b9f623db6370b22ec910461f06a52',
'InstanceList': '2.2-ff71772c7bf6d72f6ef6eee0199fb1c9',
'InstanceMapping': '1.0-65de80c491f54d19374703c0753c4d47',
'InstanceMappingList': '1.0-9e982e3de1613b9ada85e35f69b23d47',
'InstanceNUMACell': '1.3-6991a20992c5faa57fae71a45b40241b',

17
releasenotes/notes/pagination-for-usage-a313397f9a7e9a70.yaml Normal file
View File

@ -0,0 +1,17 @@
---
features:
- |
Added microversion v2.40 which introduces pagination support for usage
with the help of new optional parameters 'limit' and 'marker'. If 'limit'
isn't provided, it will default to the configurable max limit which is
currently 1000.
::
/os-simple-tenant-usage?limit={limit}&marker={instance_uuid}
/os-simple-tenant-usage/{tenant}?limit={limit}&marker={instance_uuid}
Older microversions will not accept these new paging query parameters,
but they will start to silently limit by the max limit to encourage the
adoption of this new microversion, and circumvent the existing possibility
DoS-like usage requests on systems with thousands of instances.

1
tests-py3.txt
View File

@ -5,7 +5,6 @@ nova.tests.unit.api.openstack.compute.test_security_group_default_rules.TestSecu
nova.tests.unit.api.openstack.compute.test_security_group_default_rules.TestSecurityGroupDefaultRulesV21
nova.tests.unit.api.openstack.compute.test_security_groups.SecurityGroupsOutputTestV21
nova.tests.unit.api.openstack.compute.test_security_groups.TestSecurityGroupRulesV21
nova.tests.unit.api.openstack.compute.test_simple_tenant_usage.SimpleTenantUsageTestV21
nova.tests.unit.api.openstack.compute.test_user_data.ServersControllerCreateTest
nova.tests.unit.compute.test_compute.ComputeAPITestCase.test_create_with_base64_user_data
nova.tests.unit.compute.test_compute_cells.CellsComputeAPITestCase.test_create_with_base64_user_data