Browse Source

Magnum stats API

This change introduces a new /stats REST endpoint that
provide the following basic information;
1) Total number of clusters and nodes for the given tenant.
2) Total number of clusters and nodes across all the tenants.
Follow-up patches include more stats.

Change-Id: Iac0bf9343549de31654545d5b1fd7601e56142a7
Partially Implements blueprint magnum-stats-api
changes/01/391301/24
Vijendar Komalla 6 years ago
parent
commit
51e833137b
  1. 2
      api-ref/source/samples/versions-get-resp.json
  2. 3
      etc/magnum/policy.json
  3. 11
      magnum/api/controllers/v1/__init__.py
  4. 73
      magnum/api/controllers/v1/stats.py
  5. 3
      magnum/api/controllers/versions.py
  6. 13
      magnum/api/rest_api_version_history.rst
  7. 9
      magnum/db/api.py
  8. 18
      magnum/db/sqlalchemy/api.py
  9. 5
      magnum/objects/__init__.py
  10. 12
      magnum/objects/cluster.py
  11. 44
      magnum/objects/stats.py
  12. 3
      magnum/tests/fake_policy.py
  13. 6
      magnum/tests/unit/api/controllers/test_root.py
  14. 130
      magnum/tests/unit/api/controllers/v1/test_stats.py
  15. 20
      magnum/tests/unit/db/test_cluster.py
  16. 3
      magnum/tests/unit/objects/test_objects.py
  17. 6
      releasenotes/notes/stats-api-68bc66147ac027e6.yaml

2
api-ref/source/samples/versions-get-resp.json

@ -3,7 +3,7 @@
{
"status":"CURRENT",
"min_version":"1.1",
"max_version":"1.3",
"max_version":"1.4",
"id":"v1",
"links":[
{

3
etc/magnum/policy.json

@ -38,5 +38,6 @@
"certificate:create": "rule:admin_or_user",
"certificate:get": "rule:admin_or_user",
"magnum-service:get_all": "rule:admin_api"
"magnum-service:get_all": "rule:admin_api",
"stats:get_all": "rule:admin_or_owner"
}

11
magnum/api/controllers/v1/__init__.py

@ -30,6 +30,7 @@ from magnum.api.controllers.v1 import certificate
from magnum.api.controllers.v1 import cluster
from magnum.api.controllers.v1 import cluster_template
from magnum.api.controllers.v1 import magnum_services
from magnum.api.controllers.v1 import stats
from magnum.api.controllers import versions as ver
from magnum.api import expose
from magnum.api import http_error
@ -91,6 +92,9 @@ class V1(controllers_base.APIBase):
mservices = [link.Link]
"""Links to the magnum-services resource"""
stats = [link.Link]
"""Links to the stats resource"""
@staticmethod
def convert():
v1 = V1()
@ -141,6 +145,12 @@ class V1(controllers_base.APIBase):
pecan.request.host_url,
'mservices', '',
bookmark=True)]
v1.stats = [link.Link.make_link('self', pecan.request.host_url,
'stats', ''),
link.Link.make_link('bookmark',
pecan.request.host_url,
'stats', '',
bookmark=True)]
return v1
@ -153,6 +163,7 @@ class Controller(controllers_base.Controller):
clustertemplates = cluster_template.ClusterTemplatesController()
certificates = certificate.CertificateController()
mservices = magnum_services.MagnumServiceController()
stats = stats.StatsController()
@expose.expose(V1)
def get(self):

73
magnum/api/controllers/v1/stats.py

@ -0,0 +1,73 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pecan
from wsme import types as wtypes
from magnum.api.controllers import base
from magnum.api import expose
from magnum.common import exception
from magnum.common import policy
from magnum.i18n import _
from magnum import objects
class Stats(base.APIBase):
clusters = wtypes.IntegerType(minimum=0)
nodes = wtypes.IntegerType(minimum=0)
def __init__(self, **kwargs):
self.fields = []
for field in objects.Stats.fields:
# Skip fields we do not expose.
if not hasattr(self, field):
continue
self.fields.append(field)
setattr(self, field, kwargs.get(field, wtypes.Unset))
@classmethod
def convert(cls, rpc_stats):
return Stats(**rpc_stats.as_dict())
class StatsController(base.Controller):
"""REST controller for Stats."""
def __init__(self, **kwargs):
super(StatsController, self).__init__()
@base.Controller.api_version("1.4") # noqa
@expose.expose(Stats, wtypes.text, wtypes.text)
def get_all(self, project_id=None, type="cluster"):
"""Retrieve magnum stats.
"""
context = pecan.request.context
policy.enforce(context, 'stats:get_all', action='stats:get_all')
allowed_stats = ["cluster"]
if type.lower() not in allowed_stats:
msg = _("Invalid stats type. Allowed values are '%s'")
allowed_str = ','.join(allowed_stats)
raise exception.InvalidParameterValue(err=msg % allowed_str)
# 1.If the requester is not an admin and trying to request stats for
# different tenant, then reject the request
# 2.If the requester is not an admin and project_id was not provided,
# then return self stats
if not context.is_admin:
project_id = project_id if project_id else context.project_id
if project_id != context.project_id:
raise exception.NotAuthorized()
stats = objects.Stats.get_cluster_stats(context, project_id)
return Stats.convert(stats)

3
magnum/api/controllers/versions.py

@ -36,10 +36,11 @@ REST_API_VERSION_HISTORY = """REST API Version History:
* 1.1 - Initial version
* 1.2 - Async bay operations support
* 1.3 - Add bay rollback support
* 1.4 - Add stats API
"""
BASE_VER = '1.1'
CURRENT_MAX_VER = '1.3'
CURRENT_MAX_VER = '1.4'
class Version(object):

13
magnum/api/rest_api_version_history.rst

@ -44,3 +44,16 @@ user documentation.
For example:-
- http://XXX/v1/clusters/XXX/?rollback=True or
- http://XXX/v1/bays/XXX/?rollback=True
1.4
---
Add stats API
An admin user can get total number of clusters and nodes for a specified
tenant or for all the tenants and also a non-admin user can get self stats.
For example:-
- http://XXX/v1/stats or
- http://XXX/v1/stats?project_id=<project-id> or
- http://XXX/v1/stats?project_id=<project-id>&type=<stats-type>

9
magnum/db/api.py

@ -106,6 +106,15 @@ class Connection(object):
:returns: A cluster.
"""
@abc.abstractmethod
def get_cluster_stats(self, context, project_id):
"""Return clusters stats for the given project.
:param context: The security context
:param project_id: The project id.
:returns: clusters, nodes count.
"""
@abc.abstractmethod
def destroy_cluster(self, cluster_id):
"""Destroy a cluster and all associated interfaces.

18
magnum/db/sqlalchemy/api.py

@ -22,6 +22,7 @@ from oslo_utils import timeutils
from oslo_utils import uuidutils
from sqlalchemy.orm.exc import MultipleResultsFound
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql import func
from magnum.common import exception
import magnum.conf
@ -190,6 +191,23 @@ class Connection(api.Connection):
except NoResultFound:
raise exception.ClusterNotFound(cluster=cluster_uuid)
def get_cluster_stats(self, context, project_id=None):
query = model_query(models.Cluster)
node_count_col = models.Cluster.node_count
master_count_col = models.Cluster.master_count
ncfunc = func.sum(node_count_col + master_count_col)
if project_id:
query = query.filter_by(project_id=project_id)
nquery = query.session.query(ncfunc.label("nodes")).filter_by(
project_id=project_id)
else:
nquery = query.session.query(ncfunc.label("nodes"))
clusters = query.count()
nodes = int(nquery.one()[0]) if nquery.one()[0] else 0
return clusters, nodes
def destroy_cluster(self, cluster_id):
session = get_session()
with session.begin():

5
magnum/objects/__init__.py

@ -16,6 +16,7 @@ from magnum.objects import certificate
from magnum.objects import cluster
from magnum.objects import cluster_template
from magnum.objects import magnum_service
from magnum.objects import stats
from magnum.objects import x509keypair
@ -24,8 +25,10 @@ ClusterTemplate = cluster_template.ClusterTemplate
MagnumService = magnum_service.MagnumService
X509KeyPair = x509keypair.X509KeyPair
Certificate = certificate.Certificate
Stats = stats.Stats
__all__ = (Cluster,
ClusterTemplate,
MagnumService,
X509KeyPair,
Certificate)
Certificate,
Stats)

12
magnum/objects/cluster.py

@ -40,8 +40,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
# Rename 'bay_create_timeout' to 'create_timeout'
# Version 1.10: Added 'keypair' field
# Version 1.11: Added 'RESUME_FAILED' in status field
# Version 1.12: Added 'get_stats' method
VERSION = '1.11'
VERSION = '1.12'
dbapi = dbapi.get_instance()
@ -172,6 +173,15 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
filters=filters)
return Cluster._from_db_object_list(db_clusters, cls, context)
@base.remotable_classmethod
def get_stats(cls, context, project_id=None):
"""Return a list of Cluster objects.
:param context: Security context.
:param project_id: project id
"""
return cls.dbapi.get_cluster_stats(project_id)
@base.remotable
def create(self, context=None):
"""Create a Cluster record in the DB.

44
magnum/objects/stats.py

@ -0,0 +1,44 @@
# coding=utf-8
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_versionedobjects import fields
from magnum.db import api as dbapi
from magnum.objects import base
@base.MagnumObjectRegistry.register
class Stats(base.MagnumObject, base.MagnumObjectDictCompat):
# Version 1.0: Initial version
VERSION = '1.0'
dbapi = dbapi.get_instance()
fields = {
'clusters': fields.IntegerField(),
'nodes': fields.IntegerField(nullable=True)
}
@base.remotable_classmethod
def get_cluster_stats(cls, context, project_id=None):
"""Return cluster stats for the given project.
:param context: The security context
:param project_id: project id
"""
clusters, nodes = cls.dbapi.get_cluster_stats(context, project_id)
return cls(clusters=clusters, nodes=nodes)

3
magnum/tests/fake_policy.py

@ -52,6 +52,7 @@ policy_data = """
"certificate:create": "",
"certificate:get": "",
"magnum-service:get_all": ""
"magnum-service:get_all": "",
"stats:get_all": ""
}
"""

6
magnum/tests/unit/api/controllers/test_root.py

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/',
u'rel': u'self'}],
u'status': u'CURRENT',
u'max_version': u'1.3',
u'max_version': u'1.4',
u'min_version': u'1.1'}]}
self.v1_expected = {
@ -53,6 +53,10 @@ class TestRootController(api_base.FunctionalTest):
u'http://docs.openstack.org/developer'
'/magnum/dev/api-spec-v1.html',
u'type': u'text/html', u'rel': u'describedby'}],
u'stats': [{u'href': u'http://localhost/v1/stats/',
u'rel': u'self'},
{u'href': u'http://localhost/stats/',
u'rel': u'bookmark'}],
u'bays': [{u'href': u'http://localhost/v1/bays/',
u'rel': u'self'},
{u'href': u'http://localhost/bays/',

130
magnum/tests/unit/api/controllers/v1/test_stats.py

@ -0,0 +1,130 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from webtest.app import AppError
from magnum.tests.unit.api import base as api_base
from magnum.tests.unit.objects import utils as obj_utils
class TestStatsController(api_base.FunctionalTest):
def setUp(self):
self.base_headers = {'OpenStack-API-Version': 'container-infra 1.4'}
super(TestStatsController, self).setUp()
obj_utils.create_test_cluster_template(self.context)
def test_empty(self):
response = self.get_json('/stats', headers=self.base_headers)
expected = {u'clusters': 0, u'nodes': 0}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_all_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
response = self.get_json('/stats', headers=self.base_headers)
expected = {u'clusters': 2, u'nodes': 12}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_tenant_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
self.context.is_admin = True
response = self.get_json('/stats?project_id=234',
headers=self.base_headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_invalid_tenant_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
self.context.is_admin = True
response = self.get_json('/stats?project_id=34',
headers=self.base_headers)
expected = {u'clusters': 0, u'nodes': 0}
self.assertEqual(expected, response)
def test_get_self_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5,
master_count=1)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
response = self.get_json('/stats',
headers=headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
def test_get_self_stats_without_param(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5,
master_count=1)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
response = self.get_json('/stats',
headers=headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
def test_get_some_other_user_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
self.assertRaises(AppError,
self.get_json,
'/stats?project_id=123',
headers=headers)
def test_get_invalid_type_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
self.assertRaises(AppError,
self.get_json,
'/stats?project_id=123&type=invalid',
headers=self.base_headers)

20
magnum/tests/unit/db/test_cluster.py

@ -78,6 +78,26 @@ class DbClusterTestCase(base.DbTestCase):
self.dbapi.get_cluster_by_name,
self.context, 'clusterone')
def test_get_all_cluster_stats(self):
utils.create_test_cluster(
id=1, name='clusterone',
uuid=uuidutils.generate_uuid())
utils.create_test_cluster(
id=2, name='clustertwo',
uuid=uuidutils.generate_uuid())
ret = self.dbapi.get_cluster_stats(self.context)
self.assertEqual(ret, (2, 12))
def test_get_one_tenant_cluster_stats(self):
utils.create_test_cluster(
id=1, name='clusterone', project_id='proj1',
uuid=uuidutils.generate_uuid())
utils.create_test_cluster(
id=2, name='clustertwo', project_id='proj2',
uuid=uuidutils.generate_uuid())
ret = self.dbapi.get_cluster_stats(self.context, 'proj2')
self.assertEqual(ret, (1, 6))
def test_get_cluster_list(self):
uuids = []
for i in range(1, 6):

3
magnum/tests/unit/objects/test_objects.py

@ -355,12 +355,13 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Cluster': '1.11-d4566648f0158e45e43b0c0419814d1f',
'Cluster': '1.12-73881c0604a6c90d7ecfeb5abd380f7e',
'ClusterTemplate': '1.17-65a95ef932dd08800a83871eb3cf312b',
'Certificate': '1.1-1924dc077daa844f0f9076332ef96815',
'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',
'X509KeyPair': '1.2-d81950af36c59a71365e33ce539d24f9',
'MagnumService': '1.0-2d397ec59b0046bd5ec35cd3e06efeca',
'Stats': '1.0-73a1cd6e3c0294c932a66547faba216c',
}

6
releasenotes/notes/stats-api-68bc66147ac027e6.yaml

@ -0,0 +1,6 @@
---
features:
- This release introduces 'stats' endpoint that provide the
total number of clusters and the total number of nodes
for the given tenant and also overall stats across all
the tenants.
Loading…
Cancel
Save