From f84e8450dad7922a0973cb9f484b72eae808d095 Mon Sep 17 00:00:00 2001 From: Georgy Okrokvertskhov Date: Sat, 1 Mar 2014 00:37:51 -0800 Subject: [PATCH] Add per API call statistics This patch adds a new decorator which calls a stats collection code to collect usage information per tenant StatisticsCollection class is added a a stats storage. Add hostname to stats As Murano has multi-service deployment it is necessary to add hostname to each stats in order to understand which API service provides them. Add DB model for stats This patch adds a new DB model to keep API stats in the DB. As it is possible to have multiple Murano API servers we will keep stats for each Murano API instance Fix stats update to save data to DB Fixed issues with column names Added logic to calculate request\sec and errors\sec based on previous values Change-Id: Id5c3cdc90700563aff881e5831285a1330a2c034 Partly-Implements: blueprint api-request-stats --- muranoapi/api/v1/__init__.py | 2 + muranoapi/api/v1/deployments.py | 8 +- muranoapi/api/v1/environments.py | 12 +++ muranoapi/api/v1/services.py | 9 ++ muranoapi/api/v1/sessions.py | 7 ++ muranoapi/api/v1/statistics.py | 88 +++++++++++++++++++ muranoapi/cmd/api.py | 2 + muranoapi/common/statservice.py | 47 ++++++++++ .../versions/003_add_stats_table.py | 41 +++++++++ muranoapi/db/models.py | 23 ++++- muranoapi/db/services/stats.py | 51 +++++++++++ 11 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 muranoapi/api/v1/statistics.py create mode 100644 muranoapi/db/migrate_repo/versions/003_add_stats_table.py create mode 100644 muranoapi/db/services/stats.py diff --git a/muranoapi/api/v1/__init__.py b/muranoapi/api/v1/__init__.py index 72df718f..f95052cd 100644 --- a/muranoapi/api/v1/__init__.py +++ b/muranoapi/api/v1/__init__.py @@ -15,6 +15,8 @@ from muranoapi.db import models from muranoapi.db import session as db_session +stats = None + def get_draft(environment_id=None, session_id=None): unit = db_session.get_session() diff --git a/muranoapi/api/v1/deployments.py b/muranoapi/api/v1/deployments.py index 0b7196f8..4f4c0f3b 100644 --- a/muranoapi/api/v1/deployments.py +++ b/muranoapi/api/v1/deployments.py @@ -11,13 +11,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - from sqlalchemy import desc from webob import exc +from muranoapi.api.v1 import statistics from muranoapi.common import utils from muranoapi.db import models from muranoapi.db import session as db_session + + from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import wsgi @@ -25,8 +27,11 @@ from muranoapi.openstack.common import wsgi log = logging.getLogger(__name__) +API_NAME = 'Deployments' + class Controller(object): + @statistics.stats_count(API_NAME, 'Index') def index(self, request, environment_id): unit = db_session.get_session() verify_and_get_env(unit, environment_id, request) @@ -38,6 +43,7 @@ class Controller(object): in result] return {'deployments': deployments} + @statistics.stats_count(API_NAME, 'Statuses') def statuses(self, request, environment_id, deployment_id): unit = db_session.get_session() query = unit.query(models.Status) \ diff --git a/muranoapi/api/v1/environments.py b/muranoapi/api/v1/environments.py index 7e3a990f..18cf8825 100644 --- a/muranoapi/api/v1/environments.py +++ b/muranoapi/api/v1/environments.py @@ -15,20 +15,27 @@ from sqlalchemy import desc from webob import exc + +from muranoapi.api.v1 import statistics from muranoapi.common import utils from muranoapi.db import models from muranoapi.db.services import core_services from muranoapi.db.services import environments as envs from muranoapi.db import session as db_session + + from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import wsgi log = logging.getLogger(__name__) +API_NAME = 'Environments' class Controller(object): + + @statistics.stats_count(API_NAME, 'Index') def index(self, request): log.debug(_('Environments:List')) @@ -39,6 +46,7 @@ class Controller(object): return {"environments": environments} + @statistics.stats_count(API_NAME, 'Create') def create(self, request, body): log.debug(_('Environments:Create '.format(body))) @@ -47,6 +55,7 @@ class Controller(object): return environment.to_dict() + @statistics.stats_count(API_NAME, 'Show') def show(self, request, environment_id): log.debug(_('Environments:Show '.format(environment_id))) @@ -76,6 +85,7 @@ class Controller(object): return env + @statistics.stats_count(API_NAME, 'Update') def update(self, request, environment_id, body): log.debug(_('Environments:Update '.format(environment_id, body))) @@ -98,6 +108,7 @@ class Controller(object): return environment.to_dict() + @statistics.stats_count(API_NAME, 'Delete') def delete(self, request, environment_id): log.debug(_('Environments:Delete '.format(environment_id))) @@ -117,6 +128,7 @@ class Controller(object): envs.EnvironmentServices.delete(environment_id, request.context.auth_token) + @statistics.stats_count(API_NAME, 'LastStatus') def last(self, request, environment_id): session_id = None if hasattr(request, 'context') and request.context.session: diff --git a/muranoapi/api/v1/services.py b/muranoapi/api/v1/services.py index ff603b94..58634083 100644 --- a/muranoapi/api/v1/services.py +++ b/muranoapi/api/v1/services.py @@ -15,7 +15,10 @@ import functools as func from webob import exc + +from muranoapi.api.v1 import statistics from muranoapi.db.services import core_services + from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import wsgi @@ -24,6 +27,8 @@ from muranocommon.helpers import token_sanitizer log = logging.getLogger(__name__) +API_NAME = 'Services' + def normalize_path(f): @func.wraps(f) @@ -39,6 +44,7 @@ def normalize_path(f): class Controller(object): + @statistics.stats_count(API_NAME, 'Index') @utils.verify_env @normalize_path def get(self, request, environment_id, path): @@ -57,6 +63,7 @@ class Controller(object): raise exc.HTTPNotFound return result + @statistics.stats_count(API_NAME, 'Create') @utils.verify_session @utils.verify_env @normalize_path @@ -73,6 +80,7 @@ class Controller(object): raise exc.HTTPNotFound return result + @statistics.stats_count(API_NAME, 'Update') @utils.verify_session @utils.verify_env @normalize_path @@ -89,6 +97,7 @@ class Controller(object): raise exc.HTTPNotFound return result + @statistics.stats_count(API_NAME, 'Delete') @utils.verify_session @utils.verify_env @normalize_path diff --git a/muranoapi/api/v1/sessions.py b/muranoapi/api/v1/sessions.py index 8b6ba71d..2963cabb 100644 --- a/muranoapi/api/v1/sessions.py +++ b/muranoapi/api/v1/sessions.py @@ -14,19 +14,23 @@ from webob import exc +from muranoapi.api.v1 import statistics from muranoapi.db import models from muranoapi.db.services import environments as envs from muranoapi.db.services import sessions from muranoapi.db import session as db_session + from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import wsgi log = logging.getLogger(__name__) +API_NAME = 'Sessions' class Controller(object): + @statistics.stats_count(API_NAME, 'Create') def configure(self, request, environment_id): log.debug(_('Session:Configure '.format(environment_id))) @@ -56,6 +60,7 @@ class Controller(object): return session.to_dict() + @statistics.stats_count(API_NAME, 'Index') def show(self, request, environment_id, session_id): log.debug(_('Session:Show '.format(session_id))) @@ -85,6 +90,7 @@ class Controller(object): return session.to_dict() + @statistics.stats_count(API_NAME, 'Delete') def delete(self, request, environment_id, session_id): log.debug(_('Session:Delete '.format(session_id))) @@ -117,6 +123,7 @@ class Controller(object): return None + @statistics.stats_count(API_NAME, 'Deploy') def deploy(self, request, environment_id, session_id): log.debug(_('Session:Deploy '.format(session_id))) diff --git a/muranoapi/api/v1/statistics.py b/muranoapi/api/v1/statistics.py new file mode 100644 index 00000000..e8f8dbcd --- /dev/null +++ b/muranoapi/api/v1/statistics.py @@ -0,0 +1,88 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from muranoapi.api import v1 +from muranoapi.openstack.common import log as logging + +log = logging.getLogger(__name__) + + +class StatisticsCollection(object): + request_count = 0 + error_count = 0 + average_time = 0.0 + + requests_per_tenant = {} + errors_per_tenant = {} + + def add_api_request(self, tenant, ex_time): + self.average_time = (self.average_time * self.request_count + + ex_time) / (self.request_count + 1) + if tenant: + tenant_count = self.requests_per_tenant.get(tenant, 0) + tenant_count += 1 + self.requests_per_tenant[tenant] = tenant_count + + def add_api_error(self, tenant, ex_time): + self.average_time = (self.average_time * self.request_count + + ex_time) / (self.request_count + 1) + if tenant: + tenant_count = self.errors_per_tenant.get(tenant, 0) + tenant_count += 1 + self.errors_per_tenant[tenant] = tenant_count + + +def stats_count(api, method): + def wrapper(func): + def wrap(*args, **kwargs): + try: + ts = time.time() + result = func(*args, **kwargs) + te = time.time() + tenant = args[1].context.tenant + update_count(api, method, te - ts, + tenant) + return result + except Exception: + te = time.time() + tenant = args[1].context.tenant + update_error_count(api, method, te - te, tenant) + raise + return wrap + + return wrapper + + +def update_count(api, method, ex_time, tenant=None): + log.debug("Updating count stats for %s, %s on object %s" % (api, + method, + v1.stats)) + v1.stats.add_api_request(tenant, ex_time) + v1.stats.request_count += 1 + + +def update_error_count(api, method, ex_time, tenant=None): + log.debug("Updating count stats for %s, %s on object %s" % (api, + method, + v1.stats)) + v1.stats.add_api_error(tenant, ex_time) + v1.stats.error_count += 1 + v1.stats.request_count += 1 + + +def init_stats(): + if not v1.stats: + v1.stats = StatisticsCollection() diff --git a/muranoapi/cmd/api.py b/muranoapi/cmd/api.py index b0e8b2f6..fa5506c9 100644 --- a/muranoapi/cmd/api.py +++ b/muranoapi/cmd/api.py @@ -26,6 +26,7 @@ root = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir, os.pardir) if os.path.exists(os.path.join(root, 'muranoapi', '__init__.py')): sys.path.insert(0, root) +from muranoapi.api.v1 import statistics from muranoapi.common import config from muranoapi.common import server from muranoapi.common import statservice as stats @@ -38,6 +39,7 @@ def main(): try: config.parse_args() log.setup('muranoapi') + statistics.init_stats() launcher = service.ServiceLauncher() diff --git a/muranoapi/common/statservice.py b/muranoapi/common/statservice.py index ab22b1d4..5fda13c7 100644 --- a/muranoapi/common/statservice.py +++ b/muranoapi/common/statservice.py @@ -13,12 +13,20 @@ # under the License. import eventlet +import json +import socket +import time +from muranoapi.api import v1 +from muranoapi.api.v1 import statistics from muranoapi.common import config + +from muranoapi.db.services import stats as db_stats from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import service + conf = config.CONF.stats log = logging.getLogger(__name__) @@ -26,6 +34,10 @@ log = logging.getLogger(__name__) class StatsCollectingService(service.Service): def __init__(self): super(StatsCollectingService, self).__init__() + statistics.init_stats() + self._hostname = socket.gethostname() + self._stats_db = db_stats.Statistics() + self._prev_time = time.time() def start(self): super(StatsCollectingService, self).start() @@ -42,3 +54,38 @@ class StatsCollectingService(service.Service): def update_stats(self): log.debug(_("Updating statistic information.")) + log.debug("Stats object: %s" % v1.stats) + log.debug("Stats: Requests:%s Errors: %s Ave.Res.Time %2.4f\n" + "Per tenant: %s" % + (v1.stats.request_count, + v1.stats.error_count, + v1.stats.average_time, + v1.stats.requests_per_tenant)) + try: + stats = self._stats_db.get_stats_by_host(self._hostname) + if stats is None: + self._stats_db.create(self._hostname, + v1.stats.request_count, + v1.stats.error_count, + v1.stats.average_time, + v1.stats.requests_per_tenant) + return + + now = time.time() + t_delta = now - self._prev_time + requests_per_second = (v1.stats.request_count - + stats.request_count) / t_delta + errors_per_second = (v1.stats.error_count - + stats.error_count) / t_delta + self._prev_time = now + stats.request_count = v1.stats.request_count + stats.error_count = v1.stats.error_count + stats.average_response_time = v1.stats.average_time + stats.requests_per_tenant = json.dumps(v1.stats. + requests_per_tenant) + stats.requests_per_second = requests_per_second + stats.errors_per_second = errors_per_second + self._stats_db.update(self._hostname, stats) + except Exception as e: + log.error(_("Failed to get statistics object " + "form a database. %s" % e)) diff --git a/muranoapi/db/migrate_repo/versions/003_add_stats_table.py b/muranoapi/db/migrate_repo/versions/003_add_stats_table.py new file mode 100644 index 00000000..4d606e40 --- /dev/null +++ b/muranoapi/db/migrate_repo/versions/003_add_stats_table.py @@ -0,0 +1,41 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from sqlalchemy import schema +from sqlalchemy import types + +meta = schema.MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + stats = schema.Table( + 'apistats', + meta, + schema.Column('id', types.Integer(), primary_key=True), + schema.Column('host', types.String(80)), + schema.Column('request_count', types.BigInteger()), + schema.Column('error_count', types.BigInteger()), + schema.Column('average_response_time', types.Float()), + schema.Column('requests_per_tenant', types.Text()), + schema.Column('requests_per_second', types.Float()), + schema.Column('errors_per_second', types.Float()), + schema.Column('created', types.DateTime, nullable=False), + schema.Column('updated', types.DateTime, nullable=False)) + stats.create() + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + stats = schema.Table('apistats', meta, autoload=True) + stats.drop() diff --git a/muranoapi/db/models.py b/muranoapi/db/models.py index 7d46f908..02ae52a6 100644 --- a/muranoapi/db/models.py +++ b/muranoapi/db/models.py @@ -186,11 +186,29 @@ class Status(BASE, ModelBase): return dictionary +class ApiStats(BASE, ModelBase): + __tablename__ = 'apistats' + + id = sa.Column(sa.Integer(), primary_key=True) + host = sa.Column(sa.String(80)) + request_count = sa.Column(sa.BigInteger()) + error_count = sa.Column(sa.BigInteger()) + average_response_time = sa.Column(sa.Float()) + requests_per_tenant = sa.Column(sa.Text()) + requests_per_second = sa.Column(sa.Float()) + errors_per_second = sa.Column(sa.Float()) + + def to_dict(self): + dictionary = super(ApiStats, self).to_dict() + return dictionary + + def register_models(engine): """ Creates database tables for all models with the given engine """ - models = (Environment, Status, Session, Deployment) + models = (Environment, Status, Session, Deployment, + ApiStats) for model in models: model.metadata.create_all(engine) @@ -199,6 +217,7 @@ def unregister_models(engine): """ Drops database tables for all models with the given engine """ - models = (Environment, Status, Session, Deployment) + models = (Environment, Status, Session, Deployment, + ApiStats) for model in models: model.metadata.drop_all(engine) diff --git a/muranoapi/db/services/stats.py b/muranoapi/db/services/stats.py new file mode 100644 index 00000000..5841e465 --- /dev/null +++ b/muranoapi/db/services/stats.py @@ -0,0 +1,51 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from muranoapi.db import models as m +from muranoapi.db import session as db_session + + +class Statistics(object): + @staticmethod + def get_stats_by_id(stats_id): + db = db_session.get_session() + stats = db.query(m.ApiStats).get(stats_id) + return stats + + @staticmethod + def get_stats_by_host(host): + db = db_session.get_session() + stats = db.query(m.ApiStats).filter(m.ApiStats.host == host).first() + return stats + + @staticmethod + def create(host, request_count, error_count, + average_response_time, request_per_tenant): + stats = m.ApiStats() + stats.host = host + stats.request_count = request_count + stats.error_count = error_count + stats.average_response_time = average_response_time + stats.request_per_tenant = request_per_tenant + stats.request_per_second = 0.0 + stats.errors_per_second = 0.0 + + db = db_session.get_session() + stats.save(db) + + @staticmethod + def update(host, stats): + db = db_session.get_session() + stats.save(db)