openstack
/
magnum
Code Issues Proposed changes

Magnum stats API 

This change introduces a new /stats REST endpoint that
provide the following basic information;
1) Total number of clusters and nodes for the given tenant.
2) Total number of clusters and nodes across all the tenants.
Follow-up patches include more stats.

Change-Id: Iac0bf9343549de31654545d5b1fd7601e56142a7
Partially Implements blueprint magnum-stats-api
This commit is contained in:
Vijendar Komalla 2016-10-27 11:16:18 -05:00
parent 7f5e10a38f
commit 51e833137b
17 changed files with 353 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api-ref/source/samples/versions-get-resp.json
View File

@ -3,7 +3,7 @@
{
"status":"CURRENT",
"min_version":"1.1",
"max_version":"1.3",
"max_version":"1.4",
"id":"v1",
"links":[
{

3
etc/magnum/policy.json
View File

@ -38,5 +38,6 @@
"certificate:create": "rule:admin_or_user",
"certificate:get": "rule:admin_or_user",
"magnum-service:get_all": "rule:admin_api"
"magnum-service:get_all": "rule:admin_api",
"stats:get_all": "rule:admin_or_owner"
}

11
magnum/api/controllers/v1/__init__.py
View File

@ -30,6 +30,7 @@ from magnum.api.controllers.v1 import certificate
from magnum.api.controllers.v1 import cluster
from magnum.api.controllers.v1 import cluster_template
from magnum.api.controllers.v1 import magnum_services
from magnum.api.controllers.v1 import stats
from magnum.api.controllers import versions as ver
from magnum.api import expose
from magnum.api import http_error
@ -91,6 +92,9 @@ class V1(controllers_base.APIBase):
mservices = [link.Link]
"""Links to the magnum-services resource"""
stats = [link.Link]
"""Links to the stats resource"""
@staticmethod
def convert():
v1 = V1()
@ -141,6 +145,12 @@ class V1(controllers_base.APIBase):
pecan.request.host_url,
'mservices', '',
bookmark=True)]
v1.stats = [link.Link.make_link('self', pecan.request.host_url,
'stats', ''),
link.Link.make_link('bookmark',
pecan.request.host_url,
'stats', '',
bookmark=True)]
return v1
@ -153,6 +163,7 @@ class Controller(controllers_base.Controller):
clustertemplates = cluster_template.ClusterTemplatesController()
certificates = certificate.CertificateController()
mservices = magnum_services.MagnumServiceController()
stats = stats.StatsController()
@expose.expose(V1)
def get(self):

73
magnum/api/controllers/v1/stats.py Normal file
View File

@ -0,0 +1,73 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pecan
from wsme import types as wtypes
from magnum.api.controllers import base
from magnum.api import expose
from magnum.common import exception
from magnum.common import policy
from magnum.i18n import _
from magnum import objects
class Stats(base.APIBase):
clusters = wtypes.IntegerType(minimum=0)
nodes = wtypes.IntegerType(minimum=0)
def __init__(self, **kwargs):
self.fields = []
for field in objects.Stats.fields:
# Skip fields we do not expose.
if not hasattr(self, field):
continue
self.fields.append(field)
setattr(self, field, kwargs.get(field, wtypes.Unset))
@classmethod
def convert(cls, rpc_stats):
return Stats(**rpc_stats.as_dict())
class StatsController(base.Controller):
"""REST controller for Stats."""
def __init__(self, **kwargs):
super(StatsController, self).__init__()
@base.Controller.api_version("1.4") # noqa
@expose.expose(Stats, wtypes.text, wtypes.text)
def get_all(self, project_id=None, type="cluster"):
"""Retrieve magnum stats.
"""
context = pecan.request.context
policy.enforce(context, 'stats:get_all', action='stats:get_all')
allowed_stats = ["cluster"]
if type.lower() not in allowed_stats:
msg = _("Invalid stats type. Allowed values are '%s'")
allowed_str = ','.join(allowed_stats)
raise exception.InvalidParameterValue(err=msg % allowed_str)
# 1.If the requester is not an admin and trying to request stats for
# different tenant, then reject the request
# 2.If the requester is not an admin and project_id was not provided,
# then return self stats
if not context.is_admin:
project_id = project_id if project_id else context.project_id
if project_id != context.project_id:
raise exception.NotAuthorized()
stats = objects.Stats.get_cluster_stats(context, project_id)
return Stats.convert(stats)

3
magnum/api/controllers/versions.py
View File

@ -36,10 +36,11 @@ REST_API_VERSION_HISTORY = """REST API Version History:
* 1.1 - Initial version
* 1.2 - Async bay operations support
* 1.3 - Add bay rollback support
* 1.4 - Add stats API
"""
BASE_VER = '1.1'
CURRENT_MAX_VER = '1.3'
CURRENT_MAX_VER = '1.4'
class Version(object):

13
magnum/api/rest_api_version_history.rst
View File

@ -44,3 +44,16 @@ user documentation.
For example:-
- http://XXX/v1/clusters/XXX/?rollback=True or
- http://XXX/v1/bays/XXX/?rollback=True
1.4
---
Add stats API
An admin user can get total number of clusters and nodes for a specified
tenant or for all the tenants and also a non-admin user can get self stats.
For example:-
- http://XXX/v1/stats or
- http://XXX/v1/stats?project_id=<project-id> or
- http://XXX/v1/stats?project_id=<project-id>&type=<stats-type>

9
magnum/db/api.py
View File

@ -106,6 +106,15 @@ class Connection(object):
:returns: A cluster.
"""
@abc.abstractmethod
def get_cluster_stats(self, context, project_id):
"""Return clusters stats for the given project.
:param context: The security context
:param project_id: The project id.
:returns: clusters, nodes count.
"""
@abc.abstractmethod
def destroy_cluster(self, cluster_id):
"""Destroy a cluster and all associated interfaces.

18
magnum/db/sqlalchemy/api.py
View File

@ -22,6 +22,7 @@ from oslo_utils import timeutils
from oslo_utils import uuidutils
from sqlalchemy.orm.exc import MultipleResultsFound
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql import func
from magnum.common import exception
import magnum.conf
@ -190,6 +191,23 @@ class Connection(api.Connection):
except NoResultFound:
raise exception.ClusterNotFound(cluster=cluster_uuid)
def get_cluster_stats(self, context, project_id=None):
query = model_query(models.Cluster)
node_count_col = models.Cluster.node_count
master_count_col = models.Cluster.master_count
ncfunc = func.sum(node_count_col + master_count_col)
if project_id:
query = query.filter_by(project_id=project_id)
nquery = query.session.query(ncfunc.label("nodes")).filter_by(
project_id=project_id)
else:
nquery = query.session.query(ncfunc.label("nodes"))
clusters = query.count()
nodes = int(nquery.one()[0]) if nquery.one()[0] else 0
return clusters, nodes
def destroy_cluster(self, cluster_id):
session = get_session()
with session.begin():

5
magnum/objects/__init__.py
View File

@ -16,6 +16,7 @@ from magnum.objects import certificate
from magnum.objects import cluster
from magnum.objects import cluster_template
from magnum.objects import magnum_service
from magnum.objects import stats
from magnum.objects import x509keypair
@ -24,8 +25,10 @@ ClusterTemplate = cluster_template.ClusterTemplate
MagnumService = magnum_service.MagnumService
X509KeyPair = x509keypair.X509KeyPair
Certificate = certificate.Certificate
Stats = stats.Stats
__all__ = (Cluster,
ClusterTemplate,
MagnumService,
X509KeyPair,
Certificate)
Certificate,
Stats)

12
magnum/objects/cluster.py
View File

@ -40,8 +40,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
# Rename 'bay_create_timeout' to 'create_timeout'
# Version 1.10: Added 'keypair' field
# Version 1.11: Added 'RESUME_FAILED' in status field
# Version 1.12: Added 'get_stats' method
VERSION = '1.11'
VERSION = '1.12'
dbapi = dbapi.get_instance()
@ -172,6 +173,15 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
filters=filters)
return Cluster._from_db_object_list(db_clusters, cls, context)
@base.remotable_classmethod
def get_stats(cls, context, project_id=None):
"""Return a list of Cluster objects.
:param context: Security context.
:param project_id: project id
"""
return cls.dbapi.get_cluster_stats(project_id)
@base.remotable
def create(self, context=None):
"""Create a Cluster record in the DB.

44
magnum/objects/stats.py Normal file
View File

@ -0,0 +1,44 @@
# coding=utf-8
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_versionedobjects import fields
from magnum.db import api as dbapi
from magnum.objects import base
@base.MagnumObjectRegistry.register
class Stats(base.MagnumObject, base.MagnumObjectDictCompat):
# Version 1.0: Initial version
VERSION = '1.0'
dbapi = dbapi.get_instance()
fields = {
'clusters': fields.IntegerField(),
'nodes': fields.IntegerField(nullable=True)
}
@base.remotable_classmethod
def get_cluster_stats(cls, context, project_id=None):
"""Return cluster stats for the given project.
:param context: The security context
:param project_id: project id
"""
clusters, nodes = cls.dbapi.get_cluster_stats(context, project_id)
return cls(clusters=clusters, nodes=nodes)

3
magnum/tests/fake_policy.py
View File

@ -52,6 +52,7 @@ policy_data = """
"certificate:create": "",
"certificate:get": "",
"magnum-service:get_all": ""
"magnum-service:get_all": "",
"stats:get_all": ""
}
"""

6
magnum/tests/unit/api/controllers/test_root.py
View File

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/',
u'rel': u'self'}],
u'status': u'CURRENT',
u'max_version': u'1.3',
u'max_version': u'1.4',
u'min_version': u'1.1'}]}
self.v1_expected = {
@ -53,6 +53,10 @@ class TestRootController(api_base.FunctionalTest):
u'http://docs.openstack.org/developer'
'/magnum/dev/api-spec-v1.html',
u'type': u'text/html', u'rel': u'describedby'}],
u'stats': [{u'href': u'http://localhost/v1/stats/',
u'rel': u'self'},
{u'href': u'http://localhost/stats/',
u'rel': u'bookmark'}],
u'bays': [{u'href': u'http://localhost/v1/bays/',
u'rel': u'self'},
{u'href': u'http://localhost/bays/',

130
magnum/tests/unit/api/controllers/v1/test_stats.py Normal file
View File

@ -0,0 +1,130 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from webtest.app import AppError
from magnum.tests.unit.api import base as api_base
from magnum.tests.unit.objects import utils as obj_utils
class TestStatsController(api_base.FunctionalTest):
def setUp(self):
self.base_headers = {'OpenStack-API-Version': 'container-infra 1.4'}
super(TestStatsController, self).setUp()
obj_utils.create_test_cluster_template(self.context)
def test_empty(self):
response = self.get_json('/stats', headers=self.base_headers)
expected = {u'clusters': 0, u'nodes': 0}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_all_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
response = self.get_json('/stats', headers=self.base_headers)
expected = {u'clusters': 2, u'nodes': 12}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_tenant_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
self.context.is_admin = True
response = self.get_json('/stats?project_id=234',
headers=self.base_headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
@mock.patch("magnum.common.policy.enforce")
@mock.patch("magnum.common.context.make_context")
def test_admin_get_invalid_tenant_stats(self, mock_context, mock_policy):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2')
self.context.is_admin = True
response = self.get_json('/stats?project_id=34',
headers=self.base_headers)
expected = {u'clusters': 0, u'nodes': 0}
self.assertEqual(expected, response)
def test_get_self_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5,
master_count=1)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
response = self.get_json('/stats',
headers=headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
def test_get_self_stats_without_param(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5,
master_count=1)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
response = self.get_json('/stats',
headers=headers)
expected = {u'clusters': 1, u'nodes': 6}
self.assertEqual(expected, response)
def test_get_some_other_user_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
obj_utils.create_test_cluster(self.context,
project_id=234,
uuid='uuid2',
node_count=5)
headers = self.base_headers.copy()
headers['X-Project-Id'] = '234'
self.assertRaises(AppError,
self.get_json,
'/stats?project_id=123',
headers=headers)
def test_get_invalid_type_stats(self):
obj_utils.create_test_cluster(self.context,
project_id=123,
uuid='uuid1')
self.assertRaises(AppError,
self.get_json,
'/stats?project_id=123&type=invalid',
headers=self.base_headers)

20
magnum/tests/unit/db/test_cluster.py
View File

@ -78,6 +78,26 @@ class DbClusterTestCase(base.DbTestCase):
self.dbapi.get_cluster_by_name,
self.context, 'clusterone')
def test_get_all_cluster_stats(self):
utils.create_test_cluster(
id=1, name='clusterone',
uuid=uuidutils.generate_uuid())
utils.create_test_cluster(
id=2, name='clustertwo',
uuid=uuidutils.generate_uuid())
ret = self.dbapi.get_cluster_stats(self.context)
self.assertEqual(ret, (2, 12))
def test_get_one_tenant_cluster_stats(self):
utils.create_test_cluster(
id=1, name='clusterone', project_id='proj1',
uuid=uuidutils.generate_uuid())
utils.create_test_cluster(
id=2, name='clustertwo', project_id='proj2',
uuid=uuidutils.generate_uuid())
ret = self.dbapi.get_cluster_stats(self.context, 'proj2')
self.assertEqual(ret, (1, 6))
def test_get_cluster_list(self):
uuids = []
for i in range(1, 6):

3
magnum/tests/unit/objects/test_objects.py
View File

@ -355,12 +355,13 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Cluster': '1.11-d4566648f0158e45e43b0c0419814d1f',
'Cluster': '1.12-73881c0604a6c90d7ecfeb5abd380f7e',
'ClusterTemplate': '1.17-65a95ef932dd08800a83871eb3cf312b',
'Certificate': '1.1-1924dc077daa844f0f9076332ef96815',
'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',
'X509KeyPair': '1.2-d81950af36c59a71365e33ce539d24f9',
'MagnumService': '1.0-2d397ec59b0046bd5ec35cd3e06efeca',
'Stats': '1.0-73a1cd6e3c0294c932a66547faba216c',
}

6
releasenotes/notes/stats-api-68bc66147ac027e6.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- This release introduces 'stats' endpoint that provide the
total number of clusters and the total number of nodes
for the given tenant and also overall stats across all
the tenants.