Add a v2 API endpoint to get scope state

This adds a v2 API endpoint allowing to retrieve the state of several
scopes. It supports pagination and various filter.

Depends-On: https://review.opendev.org/#/c/658072
Change-Id: I3cb7f0554f7794eaaf2e7c35db6c36254bff96db
Story: 2005395
Task: 30789
This commit is contained in:
Luka Peschke 2019-05-07 10:35:43 +02:00
parent 24f15bdc51
commit cb540872e8
22 changed files with 476 additions and 235 deletions

View File

@ -32,7 +32,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({
API_MODULES = [
'cloudkitty.api.v2.example',
'cloudkitty.api.v2.scope',
]

View File

@ -1,68 +0,0 @@
# Copyright 2018 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import flask
import voluptuous
from werkzeug import exceptions as http_exceptions
from cloudkitty.api.v2 import base
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty.common import policy
class Example(base.BaseResource):
@api_utils.add_output_schema({
voluptuous.Required(
'message',
default='This is an example endpoint',
): api_utils.get_string_type(),
})
def get(self):
policy.authorize(flask.request.context, 'example:get_example', {})
return {}
@api_utils.add_input_schema('query', {
voluptuous.Required('fruit'): api_utils.SingleQueryParam(str),
})
def put(self, fruit=None):
policy.authorize(flask.request.context, 'example:submit_fruit', {})
if not fruit:
raise http_exceptions.BadRequest(
'You must submit a fruit',
)
if fruit not in ['banana', 'strawberry']:
raise http_exceptions.Forbidden(
'You submitted a forbidden fruit',
)
return {
'message': 'Your fruit is a ' + fruit,
}
@api_utils.add_input_schema('body', {
voluptuous.Required('fruit'): api_utils.get_string_type(),
})
def post(self, fruit=None):
policy.authorize(flask.request.context, 'example:submit_fruit', {})
if not fruit:
raise http_exceptions.BadRequest(
'You must submit a fruit',
)
if fruit not in ['banana', 'strawberry']:
raise http_exceptions.Forbidden(
'You submitted a forbidden fruit',
)
return {
'message': 'Your fruit is a ' + fruit,
}

View File

@ -1,4 +1,4 @@
# Copyright 2018 Objectif Libre
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,10 +16,10 @@ from cloudkitty.api.v2 import utils as api_utils
def init(app):
api_utils.do_init(app, 'example', [
api_utils.do_init(app, 'scope', [
{
'module': __name__ + '.' + 'example',
'resource_class': 'Example',
'module': __name__ + '.' + 'state',
'resource_class': 'ScopeState',
'url': '',
},
])

View File

@ -0,0 +1,77 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import flask
import voluptuous
from werkzeug import exceptions as http_exceptions
from cloudkitty.api.v2 import base
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty.common import policy
from cloudkitty import storage_state
class ScopeState(base.BaseResource):
@api_utils.paginated
@api_utils.add_input_schema('query', {
voluptuous.Optional('scope_id', default=[]):
api_utils.MultiQueryParam(str),
voluptuous.Optional('scope_key', default=[]):
api_utils.MultiQueryParam(str),
voluptuous.Optional('fetcher', default=[]):
api_utils.MultiQueryParam(str),
voluptuous.Optional('collector', default=[]):
api_utils.MultiQueryParam(str),
})
@api_utils.add_output_schema({'results': [{
voluptuous.Required('scope_id'): api_utils.get_string_type(),
voluptuous.Required('scope_key'): api_utils.get_string_type(),
voluptuous.Required('fetcher'): api_utils.get_string_type(),
voluptuous.Required('collector'): api_utils.get_string_type(),
voluptuous.Required('state'): api_utils.get_string_type(),
}]})
def get(self,
offset=0,
limit=100,
scope_id=None,
scope_key=None,
fetcher=None,
collector=None):
policy.authorize(
flask.request.context,
'scope:get_state',
{'tenant_id': scope_id or flask.request.context.project_id}
)
results = storage_state.StateManager().get_all(
identifier=scope_id,
scope_key=scope_key,
fetcher=fetcher,
collector=collector,
offset=offset,
limit=limit,
)
if len(results) < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
return {
'results': [{
'scope_id': r.identifier,
'scope_key': r.scope_key,
'fetcher': r.fetcher,
'collector': r.collector,
'state': str(r.state),
} for r in results]
}

View File

@ -13,6 +13,7 @@
# under the License.
#
import importlib
import itertools
import flask
import flask_restful
@ -48,10 +49,32 @@ class SingleQueryParam(object):
return self._validate(output)
class MultiQueryParam(object):
"""Voluptuous validator allowing to validate multiple query parameters.
This validator splits comma-separated query parameter into lists,
verifies their type and returns it directly, instead of returning a list
containing a single element.
Note that this validator uses ``voluptuous.Coerce`` internally and thus
should not be used together with ``api_utils.get_string_type`` in python2.
:param param_type: Type of the query parameter
"""
def __init__(self, param_type):
self._validate = lambda x: list(map(voluptuous.Coerce(param_type), x))
def __call__(self, v):
if not isinstance(v, list):
v = [v]
output = itertools.chain(*[elem.split(',') for elem in v])
return self._validate(output)
def add_input_schema(location, schema):
"""Add a voluptuous schema validation on a method's input
Takes a dict which can be converted to a volptuous schema as parameter,
Takes a dict which can be converted to a voluptuous schema as parameter,
and validates the parameters with this schema. The "location" parameter
is used to specify the parameters' location. Note that for query
parameters, a ``MultiDict`` is returned by Flask. Thus, each dict key will
@ -66,11 +89,11 @@ def add_input_schema(location, schema):
return fruit
To accept a list of query parameters, the following syntax can be used::
To accept a list of query parameters, a ``MultiQueryParam`` can be used::
from cloudkitty.api.v2 import utils as api_utils
@api_utils.add_input_schema('query', {
voluptuous.Required('fruit'): [str],
voluptuous.Required('fruit'): api_utils.MultiQueryParam(str),
})
def put(self, fruit=[]):
for f in fruit:
@ -95,7 +118,9 @@ def add_input_schema(location, schema):
if location == 'body':
args = flask.request.get_json()
elif location == 'query':
args = dict(flask.request.args)
# NOTE(lpeschke): issues with to_dict in python3.7,
# see https://github.com/pallets/werkzeug/issues/1379
args = dict(flask.request.args.lists())
try:
# ...here [2/2]
kwargs.update(wrap.input_schema(args))

View File

@ -21,7 +21,7 @@ from cloudkitty.common.policies.v1 import info as v1_info
from cloudkitty.common.policies.v1 import rating as v1_rating
from cloudkitty.common.policies.v1 import report as v1_report
from cloudkitty.common.policies.v1 import storage as v1_storage
from cloudkitty.common.policies.v2 import example as v2_example
from cloudkitty.common.policies.v2 import scope as v2_scope
def list_rules():
@ -32,5 +32,5 @@ def list_rules():
v1_rating.list_rules(),
v1_report.list_rules(),
v1_storage.list_rules(),
v2_example.list_rules(),
v2_scope.list_rules(),
)

View File

@ -1,4 +1,4 @@
# Copyright 2018 Objectif Libre
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,21 +16,16 @@ from oslo_policy import policy
from cloudkitty.common.policies import base
example_policies = [
scope_policies = [
policy.DocumentedRuleDefault(
name='example:get_example',
check_str=base.UNPROTECTED,
description='Get an example message',
operations=[{'path': '/v2/example',
name='scope:get_state',
check_str=base.ROLE_ADMIN,
description='Get the state of one or several scopes',
operations=[{'path': '/v2/scope',
'method': 'GET'}]),
policy.DocumentedRuleDefault(
name='example:submit_fruit',
check_str=base.UNPROTECTED,
description='Submit a fruit',
operations=[{'path': '/v2/example',
'method': 'POST'}]),
]
def list_rules():
return example_policies
return scope_policies

View File

@ -40,6 +40,47 @@ class StateManager(object):
model = models.IdentifierState
def get_all(self,
identifier=None,
fetcher=None,
collector=None,
scope_key=None,
limit=100, offset=0):
"""Returns the state of all scopes.
This function returns the state of all scopes with support for optional
filters.
:param identifier: optional scope identifiers to filter on
:type identifier: list
:param fetcher: optional scope fetchers to filter on
:type fetcher: list
:param collector: optional collectors to filter on
:type collector: list
:param fetcher: optional fetchers to filter on
:type fetcher: list
:param scope_key: optional scope_keys to filter on
:type scope_key: list
"""
session = db.get_session()
session.begin()
q = utils.model_query(self.model, session)
if identifier:
q = q.filter(self.model.identifier.in_(identifier))
if fetcher:
q = q.filter(self.model.fetcher.in_(fetcher))
if collector:
q = q.filter(self.model.collector.in_(collector))
if scope_key:
q = q.filter(self.model.scope_key.in_(scope_key))
q = q.offset(offset).limit(limit)
r = q.all()
session.close()
return r
def _get_db_item(self, session, identifier,
fetcher=None, collector=None, scope_key=None):
fetcher = fetcher or CONF.fetcher.backend

View File

@ -16,6 +16,7 @@ import flask
import mock
import voluptuous
from werkzeug.exceptions import BadRequest
from werkzeug import MultiDict
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty import tests
@ -27,9 +28,9 @@ class ApiUtilsDoInitTest(tests.TestCase):
app = flask.Flask('cloudkitty')
resources = [
{
'module': 'cloudkitty.api.v2.example.example',
'resource_class': 'Example',
'url': '/example',
'module': 'cloudkitty.api.v2.scope.state',
'resource_class': 'ScopeState',
'url': '/scope',
},
]
api_utils.do_init(app, 'example', resources)
@ -95,18 +96,18 @@ class AddInputSchemaTest(tests.TestCase):
self.assertEqual(2, len(test_func.input_schema.schema.keys()))
with mock.patch('flask.request') as m:
m.args = {}
m.args = MultiDict({})
test_func(self)
m.args = {'offset': 0, 'limit': 100}
m.args = MultiDict({'offset': 0, 'limit': 100})
test_func(self)
m.args = {'offset': 1}
m.args = MultiDict({'offset': 1})
self.assertRaises(AssertionError, test_func, self)
m.args = {'limit': 99}
m.args = MultiDict({'limit': 99})
self.assertRaises(AssertionError, test_func, self)
m.args = {'offset': -1}
m.args = MultiDict({'offset': -1})
self.assertRaises(BadRequest, test_func, self)
m.args = {'limit': 0}
m.args = MultiDict({'limit': 0})
self.assertRaises(BadRequest, test_func, self)
def test_simple_add_input_schema_query(self):
@ -123,9 +124,9 @@ class AddInputSchemaTest(tests.TestCase):
list(test_func.input_schema.schema.keys())[0], 'arg_one')
with mock.patch('flask.request') as m:
m.args = {}
m.args = MultiDict({})
test_func(self)
m.args = {'arg_one': 'one'}
m.args = MultiDict({'arg_one': 'one'})
test_func(self)
def test_simple_add_input_schema_body(self):

View File

@ -13,9 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import abc
import datetime
import decimal
import os
@ -42,6 +41,7 @@ from cloudkitty import messaging
from cloudkitty import rating
from cloudkitty import storage
from cloudkitty.storage.v1.sqlalchemy import models
from cloudkitty import storage_state
from cloudkitty import tests
from cloudkitty.tests import utils as test_utils
from cloudkitty import utils as ck_utils
@ -369,6 +369,33 @@ class NowStorageDataFixture(BaseStorageDataFixture):
self.storage.push(data, project_id)
class ScopeStateFixture(fixture.GabbiFixture):
def start_fixture(self):
self.sm = storage_state.StateManager()
self.sm.init()
data = [
('aaaa', datetime.datetime(2019, 1, 1), 'fet1', 'col1', 'key1'),
('bbbb', datetime.datetime(2019, 2, 2), 'fet1', 'col1', 'key2'),
('cccc', datetime.datetime(2019, 3, 3), 'fet1', 'col2', 'key1'),
('dddd', datetime.datetime(2019, 4, 4), 'fet1', 'col2', 'key2'),
('eeee', datetime.datetime(2019, 5, 5), 'fet2', 'col1', 'key1'),
('ffff', datetime.datetime(2019, 6, 6), 'fet2', 'col1', 'key2'),
('gggg', datetime.datetime(2019, 6, 6), 'fet2', 'col2', 'key1'),
('hhhh', datetime.datetime(2019, 6, 6), 'fet2', 'col2', 'key2'),
]
for d in data:
self.sm.set_state(
d[0], d[1], fetcher=d[2], collector=d[3], scope_key=d[4])
def stop_fixture(self):
session = db.get_session()
q = utils.model_query(
self.sm.model,
session)
q.delete()
class CORSConfigFixture(fixture.GabbiFixture):
"""Inject mock configuration for the CORS middleware."""

View File

@ -1,40 +0,0 @@
fixtures:
- ConfigFixtureStorageV2
tests:
- name: get an example resource
url: /v2/example
status: 200
response_json_paths:
$.message: This is an example endpoint
- name: submit a banana
url: /v2/example
status: 200
method: POST
request_headers:
content-type: application/json
data:
fruit: banana
response_json_paths:
$.message: Your fruit is a banana
- name: submit a forbidden fruit
url: /v2/example
status: 403
method: POST
request_headers:
content-type: application/json
data:
fruit: forbidden
response_json_paths:
$.message: You submitted a forbidden fruit
- name: submit invalid data
url: /v2/example
status: 400
method: POST
request_headers:
content-type: application/json
data:
invalid: invalid

View File

@ -0,0 +1,113 @@
fixtures:
- ConfigFixtureStorageV2
- ScopeStateFixture
tests:
- name: Get all scopes
url: /v2/scope
status: 200
response_json_paths:
$.results.`len`: 8
$.results.[0].scope_id: aaaa
- name: Get all scopes with limit
url: /v2/scope
status: 200
query_parameters:
limit: 2
response_json_paths:
$.results.`len`: 2
$.results.[0].scope_id: aaaa
$.results.[1].scope_id: bbbb
$.results.[*].collector: [col1, col1]
$.results.[*].fetcher: [fet1, fet1]
- name: Get all scopes with limit and offset
url: /v2/scope
status: 200
query_parameters:
limit: 2
offset: 2
response_json_paths:
$.results.`len`: 2
$.results.[0].scope_id: cccc
$.results.[1].scope_id: dddd
$.results.[*].collector: [col2, col2]
$.results.[*].fetcher: [fet1, fet1]
- name: Get all scopes with offset off bounds
url: /v2/scope
status: 404
query_parameters:
limit: 2
offset: 20
- name: Get all scopes filter on collector
url: /v2/scope
status: 200
query_parameters:
collector: col2
response_json_paths:
$.results.`len`: 4
$.results.[0].scope_id: cccc
$.results.[1].scope_id: dddd
$.results.[2].scope_id: gggg
$.results.[3].scope_id: hhhh
- name: Get all scopes filter on collector and fetcher
url: /v2/scope
status: 200
query_parameters:
collector: col2
fetcher: fet2
response_json_paths:
$.results.`len`: 2
$.results.[0].scope_id: gggg
$.results.[1].scope_id: hhhh
- name: Get all scopes filter on several collectors and one fetcher
url: /v2/scope
status: 200
query_parameters:
collector: [col2, col1]
fetcher: fet2
response_json_paths:
$.results.`len`: 4
$.results.[2].scope_id: gggg
$.results.[3].scope_id: hhhh
- name: Get all scopes filter on several comma separated collectors and one fetcher
url: /v2/scope
status: 200
query_parameters:
collector: "col2,col1"
fetcher: fet2
response_json_paths:
$.results.`len`: 4
$.results.[2].scope_id: gggg
$.results.[3].scope_id: hhhh
- name: Get all scopes filter on several collectors and several keys
url: /v2/scope
status: 200
query_parameters:
collector: [col2, col1]
scope_key: [key1, key2]
response_json_paths:
$.results.`len`: 8
$.results[0].scope_id: aaaa
- name: Get all scopes filter on scope
url: /v2/scope
status: 200
query_parameters:
scope_id: dddd
response_json_paths:
$.results.`len`: 1
$.results.[0].scope_id: dddd
- name: Get all scopes nonexistent filter
url: /v2/scopes
status: 404
query_parameters:
scope_key: nope

View File

@ -84,11 +84,7 @@
# GET /v1/storage/dataframes
#"storage:list_data_frames": "rule:admin_or_owner"
# Get an example message
# GET /v2/example
#"example:get_example": ""
# Submit a fruit
# POST /v2/example
#"example:submit_fruit": ""
# Get the state of one or several scopes
# GET /v2/scope
#"scope:get_state": "role:admin"

View File

@ -0,0 +1,25 @@
{
"results": [
{
"collector": "gnocchi",
"fetcher": "keystone",
"scope_id": "7a7e5183264644a7a79530eb56e59941",
"scope_key": "project_id",
"state": "2019-05-09 10:00:00"
},
{
"collector": "gnocchi",
"fetcher": "keystone",
"scope_id": "9084fadcbd46481788e0ad7405dcbf12",
"scope_key": "project_id",
"state": "2019-05-08 03:00:00"
},
{
"collector": "gnocchi",
"fetcher": "keystone",
"scope_id": "1f41d183fca5490ebda5c63fbaca026a",
"scope_key": "project_id",
"state": "2019-05-06 22:00:00"
}
]
}

View File

@ -1,61 +0,0 @@
================
Example endpoint
================
Get an example message
======================
Returns an example message.
.. rest_method:: GET /v2/example
Status codes
------------
.. rest_status_code:: success http_status.yml
- 200
.. rest_status_code:: error http_status.yml
- 405
Response
--------
.. rest_parameters:: example/example_parameters.yml
- msg: example_msg
Submit a fruit
==============
Returns the fruit you sent.
.. rest_method:: POST /v2/example
.. rest_parameters:: example/example_parameters.yml
- fruit: fruit
Status codes
------------
.. rest_status_code:: success http_status.yml
- 200
- 400
.. rest_status_code:: error http_status.yml
- 400
- 403: fruit_error
- 405
Response
--------
.. rest_parameters:: example/example_parameters.yml
- msg: fruit_msg

View File

@ -1,20 +0,0 @@
fruit:
in: body
description: |
A fruit. Must one of [**banana**, **strawberry**]
type: string
required: true
example_msg:
in: body
description: |
Contains "This is an example endpoint"
type: string
required: true
fruit_msg:
in: body
description: |
Contains "Your fruit is a <fruit>"
type: string
required: true

View File

@ -9,7 +9,9 @@
403:
default: Forbidden operation for the authentified user.
fruit_error: This fruit is forbidden.
404:
default: Not found
405:
default: The method is not allowed for the requested URL.

View File

@ -1,3 +1,3 @@
.. rest_expand_all::
.. include:: example/example.inc
.. include:: scope/scope.inc

View File

@ -0,0 +1,50 @@
====================
Scope state endpoint
====================
Get the status of several scopes
================================
Returns the status of several scopes.
.. rest_method:: GET /v2/scope
.. rest_parameters:: scope/scope_parameters.yml
- collector: collector
- fetcher: fetcher
- limit: limit
- offset: offset
- scope_id: scope_id
- scope_key: scope_key
Status codes
------------
.. rest_status_code:: success http_status.yml
- 200
.. rest_status_code:: error http_status.yml
- 400
- 403
- 404
- 405
Response
--------
.. rest_parameters:: scope/scope_parameters.yml
- collector: collector_resp
- fetcher: fetcher_resp
- state: state
- scope_id: scope_id_resp
- scope_key: scope_key_resp
Response Example
----------------
.. literalinclude:: ./api_samples/scope/scope_get.json
:language: javascript

View File

@ -0,0 +1,72 @@
limit:
in: query
description: |
For pagination. The maximum number of results to return.
type: int
required: false
offset: &offset
in: query
description: |
For pagination. The index of the first element that should be returned.
type: int
required: false
scope_id: &scope_id
in: query
description: |
Filter on scope.
type: string
required: false
fetcher: &fetcher
in: query
description: |
Filter on fetcher.
type: string
required: false
collector: &collector
in: query
description: |
Filter on collector.
type: string
required: false
scope_key: &scope_key
in: query
description: |
Filter on scope_key.
type: string
required: false
state:
in: body
description: |
State of the scope.
type: string
required: true
fetcher_resp:
<<: *fetcher
required: true
description: Fetcher for the given scope
in: body
scope_id_resp:
<<: *scope_id
required: true
description: Scope
in: body
collector_resp:
<<: *collector
required: true
description: Collector for the given scope
in: body
scope_key_resp:
<<: *scope_key
required: true
description: Scope key for the given scope
in: body

View File

@ -0,0 +1,6 @@
---
features:
- |
Added a v2 API endpoint allowing to retrieve the state of several scopes.
This endpoint is available via a ``GET`` request on ``/v2/scope`` and
supports filters. Admin privileges are required to use this endpoint.