Magnum stats API

This change introduces a new /stats REST endpoint that
provide the following basic information;
1) Total number of clusters and nodes for the given tenant.
2) Total number of clusters and nodes across all the tenants.
Follow-up patches include more stats.

Change-Id: Iac0bf9343549de31654545d5b1fd7601e56142a7
Partially Implements blueprint magnum-stats-api

api-ref/source/samples/versions-get-resp.json

@@ -3,7 +3,7 @@
       {
          "status":"CURRENT",
          "min_version":"1.1",
-         "max_version":"1.3",
+         "max_version":"1.4",
          "id":"v1",
          "links":[
             {

etc/magnum/policy.json

@@ -38,5 +38,6 @@
     "certificate:create": "rule:admin_or_user",
     "certificate:get": "rule:admin_or_user",
 
-    "magnum-service:get_all": "rule:admin_api"
+    "magnum-service:get_all": "rule:admin_api",
+    "stats:get_all": "rule:admin_or_owner"
 }

magnum/api/controllers/v1/__init__.py

@@ -30,6 +30,7 @@ from magnum.api.controllers.v1 import certificate
 from magnum.api.controllers.v1 import cluster
 from magnum.api.controllers.v1 import cluster_template
 from magnum.api.controllers.v1 import magnum_services
+from magnum.api.controllers.v1 import stats
 from magnum.api.controllers import versions as ver
 from magnum.api import expose
 from magnum.api import http_error
@@ -91,6 +92,9 @@ class V1(controllers_base.APIBase):
     mservices = [link.Link]
     """Links to the magnum-services resource"""
 
+    stats = [link.Link]
+    """Links to the stats resource"""
+
     @staticmethod
     def convert():
         v1 = V1()
@@ -141,6 +145,12 @@ class V1(controllers_base.APIBase):
                                             pecan.request.host_url,
                                             'mservices', '',
                                             bookmark=True)]
+        v1.stats = [link.Link.make_link('self', pecan.request.host_url,
+                                        'stats', ''),
+                    link.Link.make_link('bookmark',
+                                        pecan.request.host_url,
+                                        'stats', '',
+                                        bookmark=True)]
         return v1
 
 
@@ -153,6 +163,7 @@ class Controller(controllers_base.Controller):
     clustertemplates = cluster_template.ClusterTemplatesController()
     certificates = certificate.CertificateController()
     mservices = magnum_services.MagnumServiceController()
+    stats = stats.StatsController()
 
     @expose.expose(V1)
     def get(self):

magnum/api/controllers/v1/stats.py

@@ -0,0 +1,73 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+from wsme import types as wtypes
+
+from magnum.api.controllers import base
+from magnum.api import expose
+from magnum.common import exception
+from magnum.common import policy
+from magnum.i18n import _
+from magnum import objects
+
+
+class Stats(base.APIBase):
+
+    clusters = wtypes.IntegerType(minimum=0)
+    nodes = wtypes.IntegerType(minimum=0)
+
+    def __init__(self, **kwargs):
+        self.fields = []
+        for field in objects.Stats.fields:
+            # Skip fields we do not expose.
+            if not hasattr(self, field):
+                continue
+            self.fields.append(field)
+            setattr(self, field, kwargs.get(field, wtypes.Unset))
+
+    @classmethod
+    def convert(cls, rpc_stats):
+        return Stats(**rpc_stats.as_dict())
+
+
+class StatsController(base.Controller):
+    """REST controller for Stats."""
+    def __init__(self, **kwargs):
+        super(StatsController, self).__init__()
+
+    @base.Controller.api_version("1.4")  # noqa
+    @expose.expose(Stats, wtypes.text, wtypes.text)
+    def get_all(self, project_id=None, type="cluster"):
+        """Retrieve magnum stats.
+
+        """
+        context = pecan.request.context
+        policy.enforce(context, 'stats:get_all', action='stats:get_all')
+        allowed_stats = ["cluster"]
+
+        if type.lower() not in allowed_stats:
+            msg = _("Invalid stats type. Allowed values are '%s'")
+            allowed_str = ','.join(allowed_stats)
+            raise exception.InvalidParameterValue(err=msg % allowed_str)
+
+        # 1.If the requester is not an admin and trying to request stats for
+        # different tenant, then reject the request
+        # 2.If the requester is not an admin and project_id was not provided,
+        # then return self stats
+        if not context.is_admin:
+            project_id = project_id if project_id else context.project_id
+            if project_id != context.project_id:
+                raise exception.NotAuthorized()
+
+        stats = objects.Stats.get_cluster_stats(context, project_id)
+        return Stats.convert(stats)

magnum/api/controllers/versions.py

@@ -36,10 +36,11 @@ REST_API_VERSION_HISTORY = """REST API Version History:
     * 1.1 - Initial version
     * 1.2 - Async bay operations support
     * 1.3 - Add bay rollback support
+    * 1.4 - Add stats API
 """
 
 BASE_VER = '1.1'
-CURRENT_MAX_VER = '1.3'
+CURRENT_MAX_VER = '1.4'
 
 
 class Version(object):

magnum/api/rest_api_version_history.rst

@@ -44,3 +44,16 @@ user documentation.
   For example:-
   - http://XXX/v1/clusters/XXX/?rollback=True or
   - http://XXX/v1/bays/XXX/?rollback=True
+
+
+1.4
+---
+
+  Add stats API
+
+  An admin user can get total number of clusters and nodes for a specified
+  tenant or for all the tenants and also a non-admin user can get self stats.
+  For example:-
+  - http://XXX/v1/stats or
+  - http://XXX/v1/stats?project_id=<project-id> or
+  - http://XXX/v1/stats?project_id=<project-id>&type=<stats-type>

magnum/db/api.py

@@ -106,6 +106,15 @@ class Connection(object):
         :returns: A cluster.
         """
 
+    @abc.abstractmethod
+    def get_cluster_stats(self, context, project_id):
+        """Return clusters stats for the given project.
+
+        :param context: The security context
+        :param project_id: The project id.
+        :returns: clusters, nodes count.
+        """
+
     @abc.abstractmethod
     def destroy_cluster(self, cluster_id):
         """Destroy a cluster and all associated interfaces.

magnum/db/sqlalchemy/api.py

@@ -22,6 +22,7 @@ from oslo_utils import timeutils
 from oslo_utils import uuidutils
 from sqlalchemy.orm.exc import MultipleResultsFound
 from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.sql import func
 
 from magnum.common import exception
 import magnum.conf
@@ -190,6 +191,23 @@ class Connection(api.Connection):
         except NoResultFound:
             raise exception.ClusterNotFound(cluster=cluster_uuid)
 
+    def get_cluster_stats(self, context, project_id=None):
+        query = model_query(models.Cluster)
+        node_count_col = models.Cluster.node_count
+        master_count_col = models.Cluster.master_count
+        ncfunc = func.sum(node_count_col + master_count_col)
+
+        if project_id:
+            query = query.filter_by(project_id=project_id)
+            nquery = query.session.query(ncfunc.label("nodes")).filter_by(
+                project_id=project_id)
+        else:
+            nquery = query.session.query(ncfunc.label("nodes"))
+
+        clusters = query.count()
+        nodes = int(nquery.one()[0]) if nquery.one()[0] else 0
+        return clusters, nodes
+
     def destroy_cluster(self, cluster_id):
         session = get_session()
         with session.begin():

magnum/objects/__init__.py

@@ -16,6 +16,7 @@ from magnum.objects import certificate
 from magnum.objects import cluster
 from magnum.objects import cluster_template
 from magnum.objects import magnum_service
+from magnum.objects import stats
 from magnum.objects import x509keypair
 
 
@@ -24,8 +25,10 @@ ClusterTemplate = cluster_template.ClusterTemplate
 MagnumService = magnum_service.MagnumService
 X509KeyPair = x509keypair.X509KeyPair
 Certificate = certificate.Certificate
+Stats = stats.Stats
 __all__ = (Cluster,
            ClusterTemplate,
            MagnumService,
            X509KeyPair,
-           Certificate)
+           Certificate,
+           Stats)

magnum/objects/cluster.py

@@ -40,8 +40,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
     #              Rename 'bay_create_timeout' to 'create_timeout'
     # Version 1.10: Added 'keypair' field
     # Version 1.11: Added 'RESUME_FAILED' in status field
+    # Version 1.12: Added 'get_stats' method
 
-    VERSION = '1.11'
+    VERSION = '1.12'
 
     dbapi = dbapi.get_instance()
 
@@ -172,6 +173,15 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
                                                  filters=filters)
         return Cluster._from_db_object_list(db_clusters, cls, context)
 
+    @base.remotable_classmethod
+    def get_stats(cls, context, project_id=None):
+        """Return a list of Cluster objects.
+
+        :param context: Security context.
+        :param project_id: project id
+        """
+        return cls.dbapi.get_cluster_stats(project_id)
+
     @base.remotable
     def create(self, context=None):
         """Create a Cluster record in the DB.

magnum/objects/stats.py

@@ -0,0 +1,44 @@
+# coding=utf-8
+#
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_versionedobjects import fields
+
+from magnum.db import api as dbapi
+from magnum.objects import base
+
+
+@base.MagnumObjectRegistry.register
+class Stats(base.MagnumObject, base.MagnumObjectDictCompat):
+    # Version 1.0: Initial version
+
+    VERSION = '1.0'
+
+    dbapi = dbapi.get_instance()
+
+    fields = {
+        'clusters': fields.IntegerField(),
+        'nodes': fields.IntegerField(nullable=True)
+    }
+
+    @base.remotable_classmethod
+    def get_cluster_stats(cls, context, project_id=None):
+        """Return cluster stats for the given project.
+
+        :param context: The security context
+        :param project_id: project id
+        """
+        clusters, nodes = cls.dbapi.get_cluster_stats(context, project_id)
+        return cls(clusters=clusters, nodes=nodes)

magnum/tests/fake_policy.py

@@ -52,6 +52,7 @@ policy_data = """
     "certificate:create": "",
     "certificate:get": "",
 
-    "magnum-service:get_all": ""
+    "magnum-service:get_all": "",
+    "stats:get_all": ""
 }
 """

magnum/tests/unit/api/controllers/test_root.py

@@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
                                [{u'href': u'http://localhost/v1/',
                                  u'rel': u'self'}],
                            u'status': u'CURRENT',
-                           u'max_version': u'1.3',
+                           u'max_version': u'1.4',
                            u'min_version': u'1.1'}]}
 
         self.v1_expected = {
@@ -53,6 +53,10 @@ class TestRootController(api_base.FunctionalTest):
                         u'http://docs.openstack.org/developer'
                         '/magnum/dev/api-spec-v1.html',
                         u'type': u'text/html', u'rel': u'describedby'}],
+            u'stats': [{u'href': u'http://localhost/v1/stats/',
+                       u'rel': u'self'},
+                       {u'href': u'http://localhost/stats/',
+                        u'rel': u'bookmark'}],
             u'bays': [{u'href': u'http://localhost/v1/bays/',
                        u'rel': u'self'},
                       {u'href': u'http://localhost/bays/',

magnum/tests/unit/api/controllers/v1/test_stats.py

@@ -0,0 +1,130 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import mock
+
+from webtest.app import AppError
+
+from magnum.tests.unit.api import base as api_base
+from magnum.tests.unit.objects import utils as obj_utils
+
+
+class TestStatsController(api_base.FunctionalTest):
+
+    def setUp(self):
+        self.base_headers = {'OpenStack-API-Version': 'container-infra 1.4'}
+        super(TestStatsController, self).setUp()
+        obj_utils.create_test_cluster_template(self.context)
+
+    def test_empty(self):
+        response = self.get_json('/stats', headers=self.base_headers)
+        expected = {u'clusters': 0, u'nodes': 0}
+        self.assertEqual(expected, response)
+
+    @mock.patch("magnum.common.policy.enforce")
+    @mock.patch("magnum.common.context.make_context")
+    def test_admin_get_all_stats(self, mock_context, mock_policy):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2')
+        response = self.get_json('/stats', headers=self.base_headers)
+        expected = {u'clusters': 2, u'nodes': 12}
+        self.assertEqual(expected, response)
+
+    @mock.patch("magnum.common.policy.enforce")
+    @mock.patch("magnum.common.context.make_context")
+    def test_admin_get_tenant_stats(self, mock_context, mock_policy):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2')
+        self.context.is_admin = True
+        response = self.get_json('/stats?project_id=234',
+                                 headers=self.base_headers)
+        expected = {u'clusters': 1, u'nodes': 6}
+        self.assertEqual(expected, response)
+
+    @mock.patch("magnum.common.policy.enforce")
+    @mock.patch("magnum.common.context.make_context")
+    def test_admin_get_invalid_tenant_stats(self, mock_context, mock_policy):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2')
+        self.context.is_admin = True
+        response = self.get_json('/stats?project_id=34',
+                                 headers=self.base_headers)
+        expected = {u'clusters': 0, u'nodes': 0}
+        self.assertEqual(expected, response)
+
+    def test_get_self_stats(self):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2',
+                                      node_count=5,
+                                      master_count=1)
+        headers = self.base_headers.copy()
+        headers['X-Project-Id'] = '234'
+        response = self.get_json('/stats',
+                                 headers=headers)
+        expected = {u'clusters': 1, u'nodes': 6}
+        self.assertEqual(expected, response)
+
+    def test_get_self_stats_without_param(self):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2',
+                                      node_count=5,
+                                      master_count=1)
+        headers = self.base_headers.copy()
+        headers['X-Project-Id'] = '234'
+        response = self.get_json('/stats',
+                                 headers=headers)
+        expected = {u'clusters': 1, u'nodes': 6}
+        self.assertEqual(expected, response)
+
+    def test_get_some_other_user_stats(self):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=234,
+                                      uuid='uuid2',
+                                      node_count=5)
+        headers = self.base_headers.copy()
+        headers['X-Project-Id'] = '234'
+        self.assertRaises(AppError,
+                          self.get_json,
+                          '/stats?project_id=123',
+                          headers=headers)
+
+    def test_get_invalid_type_stats(self):
+        obj_utils.create_test_cluster(self.context,
+                                      project_id=123,
+                                      uuid='uuid1')
+        self.assertRaises(AppError,
+                          self.get_json,
+                          '/stats?project_id=123&type=invalid',
+                          headers=self.base_headers)

magnum/tests/unit/db/test_cluster.py

@@ -78,6 +78,26 @@ class DbClusterTestCase(base.DbTestCase):
                           self.dbapi.get_cluster_by_name,
                           self.context, 'clusterone')
 
+    def test_get_all_cluster_stats(self):
+        utils.create_test_cluster(
+            id=1, name='clusterone',
+            uuid=uuidutils.generate_uuid())
+        utils.create_test_cluster(
+            id=2, name='clustertwo',
+            uuid=uuidutils.generate_uuid())
+        ret = self.dbapi.get_cluster_stats(self.context)
+        self.assertEqual(ret, (2, 12))
+
+    def test_get_one_tenant_cluster_stats(self):
+        utils.create_test_cluster(
+            id=1, name='clusterone', project_id='proj1',
+            uuid=uuidutils.generate_uuid())
+        utils.create_test_cluster(
+            id=2, name='clustertwo', project_id='proj2',
+            uuid=uuidutils.generate_uuid())
+        ret = self.dbapi.get_cluster_stats(self.context, 'proj2')
+        self.assertEqual(ret, (1, 6))
+
     def test_get_cluster_list(self):
         uuids = []
         for i in range(1, 6):

magnum/tests/unit/objects/test_objects.py

@@ -355,12 +355,13 @@ class TestObject(test_base.TestCase, _TestObject):
 # For more information on object version testing, read
 # http://docs.openstack.org/developer/magnum/objects.html
 object_data = {
-    'Cluster': '1.11-d4566648f0158e45e43b0c0419814d1f',
+    'Cluster': '1.12-73881c0604a6c90d7ecfeb5abd380f7e',
     'ClusterTemplate': '1.17-65a95ef932dd08800a83871eb3cf312b',
     'Certificate': '1.1-1924dc077daa844f0f9076332ef96815',
     'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',
     'X509KeyPair': '1.2-d81950af36c59a71365e33ce539d24f9',
     'MagnumService': '1.0-2d397ec59b0046bd5ec35cd3e06efeca',
+    'Stats': '1.0-73a1cd6e3c0294c932a66547faba216c',
 }
 
 

releasenotes/notes/stats-api-68bc66147ac027e6.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - This release introduces 'stats' endpoint that provide the
+    total number of clusters and the total number of nodes
+    for the given tenant and also overall stats across all
+    the tenants.