From 51e833137be46cb6e84ccdc8cab301a3295461e9 Mon Sep 17 00:00:00 2001 From: Vijendar Komalla Date: Thu, 27 Oct 2016 11:16:18 -0500 Subject: [PATCH] Magnum stats API This change introduces a new /stats REST endpoint that provide the following basic information; 1) Total number of clusters and nodes for the given tenant. 2) Total number of clusters and nodes across all the tenants. Follow-up patches include more stats. Change-Id: Iac0bf9343549de31654545d5b1fd7601e56142a7 Partially Implements blueprint magnum-stats-api --- api-ref/source/samples/versions-get-resp.json | 2 +- etc/magnum/policy.json | 3 +- magnum/api/controllers/v1/__init__.py | 11 ++ magnum/api/controllers/v1/stats.py | 73 ++++++++++ magnum/api/controllers/versions.py | 3 +- magnum/api/rest_api_version_history.rst | 13 ++ magnum/db/api.py | 9 ++ magnum/db/sqlalchemy/api.py | 18 +++ magnum/objects/__init__.py | 5 +- magnum/objects/cluster.py | 12 +- magnum/objects/stats.py | 44 ++++++ magnum/tests/fake_policy.py | 3 +- .../tests/unit/api/controllers/test_root.py | 6 +- .../unit/api/controllers/v1/test_stats.py | 130 ++++++++++++++++++ magnum/tests/unit/db/test_cluster.py | 20 +++ magnum/tests/unit/objects/test_objects.py | 3 +- .../notes/stats-api-68bc66147ac027e6.yaml | 6 + 17 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 magnum/api/controllers/v1/stats.py create mode 100644 magnum/objects/stats.py create mode 100644 magnum/tests/unit/api/controllers/v1/test_stats.py create mode 100644 releasenotes/notes/stats-api-68bc66147ac027e6.yaml diff --git a/api-ref/source/samples/versions-get-resp.json b/api-ref/source/samples/versions-get-resp.json index c6a50c7f3e..319d9bd86d 100644 --- a/api-ref/source/samples/versions-get-resp.json +++ b/api-ref/source/samples/versions-get-resp.json @@ -3,7 +3,7 @@ { "status":"CURRENT", "min_version":"1.1", - "max_version":"1.3", + "max_version":"1.4", "id":"v1", "links":[ { diff --git a/etc/magnum/policy.json b/etc/magnum/policy.json index 6a84d2f58b..ea9f7dd0b9 100644 --- a/etc/magnum/policy.json +++ b/etc/magnum/policy.json @@ -38,5 +38,6 @@ "certificate:create": "rule:admin_or_user", "certificate:get": "rule:admin_or_user", - "magnum-service:get_all": "rule:admin_api" + "magnum-service:get_all": "rule:admin_api", + "stats:get_all": "rule:admin_or_owner" } diff --git a/magnum/api/controllers/v1/__init__.py b/magnum/api/controllers/v1/__init__.py index 92ce996b00..6b8dcf5d66 100644 --- a/magnum/api/controllers/v1/__init__.py +++ b/magnum/api/controllers/v1/__init__.py @@ -30,6 +30,7 @@ from magnum.api.controllers.v1 import certificate from magnum.api.controllers.v1 import cluster from magnum.api.controllers.v1 import cluster_template from magnum.api.controllers.v1 import magnum_services +from magnum.api.controllers.v1 import stats from magnum.api.controllers import versions as ver from magnum.api import expose from magnum.api import http_error @@ -91,6 +92,9 @@ class V1(controllers_base.APIBase): mservices = [link.Link] """Links to the magnum-services resource""" + stats = [link.Link] + """Links to the stats resource""" + @staticmethod def convert(): v1 = V1() @@ -141,6 +145,12 @@ class V1(controllers_base.APIBase): pecan.request.host_url, 'mservices', '', bookmark=True)] + v1.stats = [link.Link.make_link('self', pecan.request.host_url, + 'stats', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'stats', '', + bookmark=True)] return v1 @@ -153,6 +163,7 @@ class Controller(controllers_base.Controller): clustertemplates = cluster_template.ClusterTemplatesController() certificates = certificate.CertificateController() mservices = magnum_services.MagnumServiceController() + stats = stats.StatsController() @expose.expose(V1) def get(self): diff --git a/magnum/api/controllers/v1/stats.py b/magnum/api/controllers/v1/stats.py new file mode 100644 index 0000000000..596a3cbab0 --- /dev/null +++ b/magnum/api/controllers/v1/stats.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from wsme import types as wtypes + +from magnum.api.controllers import base +from magnum.api import expose +from magnum.common import exception +from magnum.common import policy +from magnum.i18n import _ +from magnum import objects + + +class Stats(base.APIBase): + + clusters = wtypes.IntegerType(minimum=0) + nodes = wtypes.IntegerType(minimum=0) + + def __init__(self, **kwargs): + self.fields = [] + for field in objects.Stats.fields: + # Skip fields we do not expose. + if not hasattr(self, field): + continue + self.fields.append(field) + setattr(self, field, kwargs.get(field, wtypes.Unset)) + + @classmethod + def convert(cls, rpc_stats): + return Stats(**rpc_stats.as_dict()) + + +class StatsController(base.Controller): + """REST controller for Stats.""" + def __init__(self, **kwargs): + super(StatsController, self).__init__() + + @base.Controller.api_version("1.4") # noqa + @expose.expose(Stats, wtypes.text, wtypes.text) + def get_all(self, project_id=None, type="cluster"): + """Retrieve magnum stats. + + """ + context = pecan.request.context + policy.enforce(context, 'stats:get_all', action='stats:get_all') + allowed_stats = ["cluster"] + + if type.lower() not in allowed_stats: + msg = _("Invalid stats type. Allowed values are '%s'") + allowed_str = ','.join(allowed_stats) + raise exception.InvalidParameterValue(err=msg % allowed_str) + + # 1.If the requester is not an admin and trying to request stats for + # different tenant, then reject the request + # 2.If the requester is not an admin and project_id was not provided, + # then return self stats + if not context.is_admin: + project_id = project_id if project_id else context.project_id + if project_id != context.project_id: + raise exception.NotAuthorized() + + stats = objects.Stats.get_cluster_stats(context, project_id) + return Stats.convert(stats) diff --git a/magnum/api/controllers/versions.py b/magnum/api/controllers/versions.py index 6358388292..caecfab2f7 100644 --- a/magnum/api/controllers/versions.py +++ b/magnum/api/controllers/versions.py @@ -36,10 +36,11 @@ REST_API_VERSION_HISTORY = """REST API Version History: * 1.1 - Initial version * 1.2 - Async bay operations support * 1.3 - Add bay rollback support + * 1.4 - Add stats API """ BASE_VER = '1.1' -CURRENT_MAX_VER = '1.3' +CURRENT_MAX_VER = '1.4' class Version(object): diff --git a/magnum/api/rest_api_version_history.rst b/magnum/api/rest_api_version_history.rst index 7b3f3ff52f..69d4792418 100644 --- a/magnum/api/rest_api_version_history.rst +++ b/magnum/api/rest_api_version_history.rst @@ -44,3 +44,16 @@ user documentation. For example:- - http://XXX/v1/clusters/XXX/?rollback=True or - http://XXX/v1/bays/XXX/?rollback=True + + +1.4 +--- + + Add stats API + + An admin user can get total number of clusters and nodes for a specified + tenant or for all the tenants and also a non-admin user can get self stats. + For example:- + - http://XXX/v1/stats or + - http://XXX/v1/stats?project_id= or + - http://XXX/v1/stats?project_id=&type= diff --git a/magnum/db/api.py b/magnum/db/api.py index 19b70e32be..5341d7ba44 100644 --- a/magnum/db/api.py +++ b/magnum/db/api.py @@ -106,6 +106,15 @@ class Connection(object): :returns: A cluster. """ + @abc.abstractmethod + def get_cluster_stats(self, context, project_id): + """Return clusters stats for the given project. + + :param context: The security context + :param project_id: The project id. + :returns: clusters, nodes count. + """ + @abc.abstractmethod def destroy_cluster(self, cluster_id): """Destroy a cluster and all associated interfaces. diff --git a/magnum/db/sqlalchemy/api.py b/magnum/db/sqlalchemy/api.py index 4404758bd3..cb17f9ddc2 100644 --- a/magnum/db/sqlalchemy/api.py +++ b/magnum/db/sqlalchemy/api.py @@ -22,6 +22,7 @@ from oslo_utils import timeutils from oslo_utils import uuidutils from sqlalchemy.orm.exc import MultipleResultsFound from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.sql import func from magnum.common import exception import magnum.conf @@ -190,6 +191,23 @@ class Connection(api.Connection): except NoResultFound: raise exception.ClusterNotFound(cluster=cluster_uuid) + def get_cluster_stats(self, context, project_id=None): + query = model_query(models.Cluster) + node_count_col = models.Cluster.node_count + master_count_col = models.Cluster.master_count + ncfunc = func.sum(node_count_col + master_count_col) + + if project_id: + query = query.filter_by(project_id=project_id) + nquery = query.session.query(ncfunc.label("nodes")).filter_by( + project_id=project_id) + else: + nquery = query.session.query(ncfunc.label("nodes")) + + clusters = query.count() + nodes = int(nquery.one()[0]) if nquery.one()[0] else 0 + return clusters, nodes + def destroy_cluster(self, cluster_id): session = get_session() with session.begin(): diff --git a/magnum/objects/__init__.py b/magnum/objects/__init__.py index 55c9f544d8..7b3cea7a26 100644 --- a/magnum/objects/__init__.py +++ b/magnum/objects/__init__.py @@ -16,6 +16,7 @@ from magnum.objects import certificate from magnum.objects import cluster from magnum.objects import cluster_template from magnum.objects import magnum_service +from magnum.objects import stats from magnum.objects import x509keypair @@ -24,8 +25,10 @@ ClusterTemplate = cluster_template.ClusterTemplate MagnumService = magnum_service.MagnumService X509KeyPair = x509keypair.X509KeyPair Certificate = certificate.Certificate +Stats = stats.Stats __all__ = (Cluster, ClusterTemplate, MagnumService, X509KeyPair, - Certificate) + Certificate, + Stats) diff --git a/magnum/objects/cluster.py b/magnum/objects/cluster.py index 7c010b6c7a..4715f3b409 100644 --- a/magnum/objects/cluster.py +++ b/magnum/objects/cluster.py @@ -40,8 +40,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject, # Rename 'bay_create_timeout' to 'create_timeout' # Version 1.10: Added 'keypair' field # Version 1.11: Added 'RESUME_FAILED' in status field + # Version 1.12: Added 'get_stats' method - VERSION = '1.11' + VERSION = '1.12' dbapi = dbapi.get_instance() @@ -172,6 +173,15 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject, filters=filters) return Cluster._from_db_object_list(db_clusters, cls, context) + @base.remotable_classmethod + def get_stats(cls, context, project_id=None): + """Return a list of Cluster objects. + + :param context: Security context. + :param project_id: project id + """ + return cls.dbapi.get_cluster_stats(project_id) + @base.remotable def create(self, context=None): """Create a Cluster record in the DB. diff --git a/magnum/objects/stats.py b/magnum/objects/stats.py new file mode 100644 index 0000000000..0d800c8ffa --- /dev/null +++ b/magnum/objects/stats.py @@ -0,0 +1,44 @@ +# coding=utf-8 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_versionedobjects import fields + +from magnum.db import api as dbapi +from magnum.objects import base + + +@base.MagnumObjectRegistry.register +class Stats(base.MagnumObject, base.MagnumObjectDictCompat): + # Version 1.0: Initial version + + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'clusters': fields.IntegerField(), + 'nodes': fields.IntegerField(nullable=True) + } + + @base.remotable_classmethod + def get_cluster_stats(cls, context, project_id=None): + """Return cluster stats for the given project. + + :param context: The security context + :param project_id: project id + """ + clusters, nodes = cls.dbapi.get_cluster_stats(context, project_id) + return cls(clusters=clusters, nodes=nodes) diff --git a/magnum/tests/fake_policy.py b/magnum/tests/fake_policy.py index 8c445e1f26..b051e5cb15 100644 --- a/magnum/tests/fake_policy.py +++ b/magnum/tests/fake_policy.py @@ -52,6 +52,7 @@ policy_data = """ "certificate:create": "", "certificate:get": "", - "magnum-service:get_all": "" + "magnum-service:get_all": "", + "stats:get_all": "" } """ diff --git a/magnum/tests/unit/api/controllers/test_root.py b/magnum/tests/unit/api/controllers/test_root.py index 9e38a394b1..01b3693ee1 100644 --- a/magnum/tests/unit/api/controllers/test_root.py +++ b/magnum/tests/unit/api/controllers/test_root.py @@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest): [{u'href': u'http://localhost/v1/', u'rel': u'self'}], u'status': u'CURRENT', - u'max_version': u'1.3', + u'max_version': u'1.4', u'min_version': u'1.1'}]} self.v1_expected = { @@ -53,6 +53,10 @@ class TestRootController(api_base.FunctionalTest): u'http://docs.openstack.org/developer' '/magnum/dev/api-spec-v1.html', u'type': u'text/html', u'rel': u'describedby'}], + u'stats': [{u'href': u'http://localhost/v1/stats/', + u'rel': u'self'}, + {u'href': u'http://localhost/stats/', + u'rel': u'bookmark'}], u'bays': [{u'href': u'http://localhost/v1/bays/', u'rel': u'self'}, {u'href': u'http://localhost/bays/', diff --git a/magnum/tests/unit/api/controllers/v1/test_stats.py b/magnum/tests/unit/api/controllers/v1/test_stats.py new file mode 100644 index 0000000000..1cb1b2e81c --- /dev/null +++ b/magnum/tests/unit/api/controllers/v1/test_stats.py @@ -0,0 +1,130 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from webtest.app import AppError + +from magnum.tests.unit.api import base as api_base +from magnum.tests.unit.objects import utils as obj_utils + + +class TestStatsController(api_base.FunctionalTest): + + def setUp(self): + self.base_headers = {'OpenStack-API-Version': 'container-infra 1.4'} + super(TestStatsController, self).setUp() + obj_utils.create_test_cluster_template(self.context) + + def test_empty(self): + response = self.get_json('/stats', headers=self.base_headers) + expected = {u'clusters': 0, u'nodes': 0} + self.assertEqual(expected, response) + + @mock.patch("magnum.common.policy.enforce") + @mock.patch("magnum.common.context.make_context") + def test_admin_get_all_stats(self, mock_context, mock_policy): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2') + response = self.get_json('/stats', headers=self.base_headers) + expected = {u'clusters': 2, u'nodes': 12} + self.assertEqual(expected, response) + + @mock.patch("magnum.common.policy.enforce") + @mock.patch("magnum.common.context.make_context") + def test_admin_get_tenant_stats(self, mock_context, mock_policy): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2') + self.context.is_admin = True + response = self.get_json('/stats?project_id=234', + headers=self.base_headers) + expected = {u'clusters': 1, u'nodes': 6} + self.assertEqual(expected, response) + + @mock.patch("magnum.common.policy.enforce") + @mock.patch("magnum.common.context.make_context") + def test_admin_get_invalid_tenant_stats(self, mock_context, mock_policy): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2') + self.context.is_admin = True + response = self.get_json('/stats?project_id=34', + headers=self.base_headers) + expected = {u'clusters': 0, u'nodes': 0} + self.assertEqual(expected, response) + + def test_get_self_stats(self): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2', + node_count=5, + master_count=1) + headers = self.base_headers.copy() + headers['X-Project-Id'] = '234' + response = self.get_json('/stats', + headers=headers) + expected = {u'clusters': 1, u'nodes': 6} + self.assertEqual(expected, response) + + def test_get_self_stats_without_param(self): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2', + node_count=5, + master_count=1) + headers = self.base_headers.copy() + headers['X-Project-Id'] = '234' + response = self.get_json('/stats', + headers=headers) + expected = {u'clusters': 1, u'nodes': 6} + self.assertEqual(expected, response) + + def test_get_some_other_user_stats(self): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + obj_utils.create_test_cluster(self.context, + project_id=234, + uuid='uuid2', + node_count=5) + headers = self.base_headers.copy() + headers['X-Project-Id'] = '234' + self.assertRaises(AppError, + self.get_json, + '/stats?project_id=123', + headers=headers) + + def test_get_invalid_type_stats(self): + obj_utils.create_test_cluster(self.context, + project_id=123, + uuid='uuid1') + self.assertRaises(AppError, + self.get_json, + '/stats?project_id=123&type=invalid', + headers=self.base_headers) diff --git a/magnum/tests/unit/db/test_cluster.py b/magnum/tests/unit/db/test_cluster.py index 31c7cc2486..f0b0561dc0 100644 --- a/magnum/tests/unit/db/test_cluster.py +++ b/magnum/tests/unit/db/test_cluster.py @@ -78,6 +78,26 @@ class DbClusterTestCase(base.DbTestCase): self.dbapi.get_cluster_by_name, self.context, 'clusterone') + def test_get_all_cluster_stats(self): + utils.create_test_cluster( + id=1, name='clusterone', + uuid=uuidutils.generate_uuid()) + utils.create_test_cluster( + id=2, name='clustertwo', + uuid=uuidutils.generate_uuid()) + ret = self.dbapi.get_cluster_stats(self.context) + self.assertEqual(ret, (2, 12)) + + def test_get_one_tenant_cluster_stats(self): + utils.create_test_cluster( + id=1, name='clusterone', project_id='proj1', + uuid=uuidutils.generate_uuid()) + utils.create_test_cluster( + id=2, name='clustertwo', project_id='proj2', + uuid=uuidutils.generate_uuid()) + ret = self.dbapi.get_cluster_stats(self.context, 'proj2') + self.assertEqual(ret, (1, 6)) + def test_get_cluster_list(self): uuids = [] for i in range(1, 6): diff --git a/magnum/tests/unit/objects/test_objects.py b/magnum/tests/unit/objects/test_objects.py index 2b3eb81ed2..329547934c 100644 --- a/magnum/tests/unit/objects/test_objects.py +++ b/magnum/tests/unit/objects/test_objects.py @@ -355,12 +355,13 @@ class TestObject(test_base.TestCase, _TestObject): # For more information on object version testing, read # http://docs.openstack.org/developer/magnum/objects.html object_data = { - 'Cluster': '1.11-d4566648f0158e45e43b0c0419814d1f', + 'Cluster': '1.12-73881c0604a6c90d7ecfeb5abd380f7e', 'ClusterTemplate': '1.17-65a95ef932dd08800a83871eb3cf312b', 'Certificate': '1.1-1924dc077daa844f0f9076332ef96815', 'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd', 'X509KeyPair': '1.2-d81950af36c59a71365e33ce539d24f9', 'MagnumService': '1.0-2d397ec59b0046bd5ec35cd3e06efeca', + 'Stats': '1.0-73a1cd6e3c0294c932a66547faba216c', } diff --git a/releasenotes/notes/stats-api-68bc66147ac027e6.yaml b/releasenotes/notes/stats-api-68bc66147ac027e6.yaml new file mode 100644 index 0000000000..d2e25909ba --- /dev/null +++ b/releasenotes/notes/stats-api-68bc66147ac027e6.yaml @@ -0,0 +1,6 @@ +--- +features: + - This release introduces 'stats' endpoint that provide the + total number of clusters and the total number of nodes + for the given tenant and also overall stats across all + the tenants.