openstack/deb-murano
Code Issues Proposed changes

Merge "Add per API call statistics"

Browse Source
This commit is contained in:
Jenkins 2014-03-14 15:25:38 +00:00 committed by Gerrit Code Review
commit 2b0c534a0e
11 changed files with 287 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
muranoapi/api/v1/__init__.py
View File

@ -15,6 +15,8 @@
from muranoapi.db import models from muranoapi.db import models
from muranoapi.db import session as db_session from muranoapi.db import session as db_session
stats = None
def get_draft(environment_id=None, session_id=None): def get_draft(environment_id=None, session_id=None):
unit = db_session.get_session() unit = db_session.get_session()

8
muranoapi/api/v1/deployments.py
View File

@ -11,13 +11,15 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from sqlalchemy import desc from sqlalchemy import desc
from webob import exc from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.common import utils from muranoapi.common import utils
from muranoapi.db import models from muranoapi.db import models
from muranoapi.db import session as db_session from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi from muranoapi.openstack.common import wsgi
@ -25,8 +27,11 @@ from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
API_NAME = 'Deployments'
class Controller(object): class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
def index(self, request, environment_id): def index(self, request, environment_id):
unit = db_session.get_session() unit = db_session.get_session()
verify_and_get_env(unit, environment_id, request) verify_and_get_env(unit, environment_id, request)
@ -38,6 +43,7 @@ class Controller(object):
in result] in result]
return {'deployments': deployments} return {'deployments': deployments}
@statistics.stats_count(API_NAME, 'Statuses')
def statuses(self, request, environment_id, deployment_id): def statuses(self, request, environment_id, deployment_id):
unit = db_session.get_session() unit = db_session.get_session()
query = unit.query(models.Status) \ query = unit.query(models.Status) \

12
muranoapi/api/v1/environments.py
View File

@ -15,20 +15,27 @@
from sqlalchemy import desc from sqlalchemy import desc
from webob import exc from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.common import utils from muranoapi.common import utils
from muranoapi.db import models from muranoapi.db import models
from muranoapi.db.services import core_services from muranoapi.db.services import core_services
from muranoapi.db.services import environments as envs from muranoapi.db.services import environments as envs
from muranoapi.db import session as db_session from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
API_NAME = 'Environments'
class Controller(object): class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
def index(self, request): def index(self, request):
log.debug(_('Environments:List')) log.debug(_('Environments:List'))
@ -39,6 +46,7 @@ class Controller(object):
return {"environments": environments} return {"environments": environments}
@statistics.stats_count(API_NAME, 'Create')
def create(self, request, body): def create(self, request, body):
log.debug(_('Environments:Create <Body {0}>'.format(body))) log.debug(_('Environments:Create <Body {0}>'.format(body)))
@ -47,6 +55,7 @@ class Controller(object):
return environment.to_dict() return environment.to_dict()
@statistics.stats_count(API_NAME, 'Show')
def show(self, request, environment_id): def show(self, request, environment_id):
log.debug(_('Environments:Show <Id: {0}>'.format(environment_id))) log.debug(_('Environments:Show <Id: {0}>'.format(environment_id)))
@ -76,6 +85,7 @@ class Controller(object):
return env return env
@statistics.stats_count(API_NAME, 'Update')
def update(self, request, environment_id, body): def update(self, request, environment_id, body):
log.debug(_('Environments:Update <Id: {0}, ' log.debug(_('Environments:Update <Id: {0}, '
'Body: {1}>'.format(environment_id, body))) 'Body: {1}>'.format(environment_id, body)))
@ -98,6 +108,7 @@ class Controller(object):
return environment.to_dict() return environment.to_dict()
@statistics.stats_count(API_NAME, 'Delete')
def delete(self, request, environment_id): def delete(self, request, environment_id):
log.debug(_('Environments:Delete <Id: {0}>'.format(environment_id))) log.debug(_('Environments:Delete <Id: {0}>'.format(environment_id)))
@ -117,6 +128,7 @@ class Controller(object):
envs.EnvironmentServices.delete(environment_id, envs.EnvironmentServices.delete(environment_id,
request.context.auth_token) request.context.auth_token)
@statistics.stats_count(API_NAME, 'LastStatus')
def last(self, request, environment_id): def last(self, request, environment_id):
session_id = None session_id = None
if hasattr(request, 'context') and request.context.session: if hasattr(request, 'context') and request.context.session:

9
muranoapi/api/v1/services.py
View File

@ -15,7 +15,10 @@ import functools as func
from webob import exc from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.db.services import core_services from muranoapi.db.services import core_services
from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi from muranoapi.openstack.common import wsgi
@ -24,6 +27,8 @@ from muranocommon.helpers import token_sanitizer
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
API_NAME = 'Services'
def normalize_path(f): def normalize_path(f):
@func.wraps(f) @func.wraps(f)
@ -39,6 +44,7 @@ def normalize_path(f):
class Controller(object): class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
@utils.verify_env @utils.verify_env
@normalize_path @normalize_path
def get(self, request, environment_id, path): def get(self, request, environment_id, path):
@ -57,6 +63,7 @@ class Controller(object):
raise exc.HTTPNotFound raise exc.HTTPNotFound
return result return result
@statistics.stats_count(API_NAME, 'Create')
@utils.verify_session @utils.verify_session
@utils.verify_env @utils.verify_env
@normalize_path @normalize_path
@ -73,6 +80,7 @@ class Controller(object):
raise exc.HTTPNotFound raise exc.HTTPNotFound
return result return result
@statistics.stats_count(API_NAME, 'Update')
@utils.verify_session @utils.verify_session
@utils.verify_env @utils.verify_env
@normalize_path @normalize_path
@ -89,6 +97,7 @@ class Controller(object):
raise exc.HTTPNotFound raise exc.HTTPNotFound
return result return result
@statistics.stats_count(API_NAME, 'Delete')
@utils.verify_session @utils.verify_session
@utils.verify_env @utils.verify_env
@normalize_path @normalize_path

7
muranoapi/api/v1/sessions.py
View File

@ -14,19 +14,23 @@
from webob import exc from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.db import models from muranoapi.db import models
from muranoapi.db.services import environments as envs from muranoapi.db.services import environments as envs
from muranoapi.db.services import sessions from muranoapi.db.services import sessions
from muranoapi.db import session as db_session from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
API_NAME = 'Sessions'
class Controller(object): class Controller(object):
@statistics.stats_count(API_NAME, 'Create')
def configure(self, request, environment_id): def configure(self, request, environment_id):
log.debug(_('Session:Configure <EnvId: {0}>'.format(environment_id))) log.debug(_('Session:Configure <EnvId: {0}>'.format(environment_id)))
@ -56,6 +60,7 @@ class Controller(object):
return session.to_dict() return session.to_dict()
@statistics.stats_count(API_NAME, 'Index')
def show(self, request, environment_id, session_id): def show(self, request, environment_id, session_id):
log.debug(_('Session:Show <SessionId: {0}>'.format(session_id))) log.debug(_('Session:Show <SessionId: {0}>'.format(session_id)))
@ -85,6 +90,7 @@ class Controller(object):
return session.to_dict() return session.to_dict()
@statistics.stats_count(API_NAME, 'Delete')
def delete(self, request, environment_id, session_id): def delete(self, request, environment_id, session_id):
log.debug(_('Session:Delete <SessionId: {0}>'.format(session_id))) log.debug(_('Session:Delete <SessionId: {0}>'.format(session_id)))
@ -117,6 +123,7 @@ class Controller(object):
return None return None
@statistics.stats_count(API_NAME, 'Deploy')
def deploy(self, request, environment_id, session_id): def deploy(self, request, environment_id, session_id):
log.debug(_('Session:Deploy <SessionId: {0}>'.format(session_id))) log.debug(_('Session:Deploy <SessionId: {0}>'.format(session_id)))

88
muranoapi/api/v1/statistics.py Normal file
View File

@ -0,0 +1,88 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from muranoapi.api import v1
from muranoapi.openstack.common import log as logging
log = logging.getLogger(__name__)
class StatisticsCollection(object):
request_count = 0
error_count = 0
average_time = 0.0
requests_per_tenant = {}
errors_per_tenant = {}
def add_api_request(self, tenant, ex_time):
self.average_time = (self.average_time * self.request_count +
ex_time) / (self.request_count + 1)
if tenant:
tenant_count = self.requests_per_tenant.get(tenant, 0)
tenant_count += 1
self.requests_per_tenant[tenant] = tenant_count
def add_api_error(self, tenant, ex_time):
self.average_time = (self.average_time * self.request_count +
ex_time) / (self.request_count + 1)
if tenant:
tenant_count = self.errors_per_tenant.get(tenant, 0)
tenant_count += 1
self.errors_per_tenant[tenant] = tenant_count
def stats_count(api, method):
def wrapper(func):
def wrap(*args, **kwargs):
try:
ts = time.time()
result = func(*args, **kwargs)
te = time.time()
tenant = args[1].context.tenant
update_count(api, method, te - ts,
tenant)
return result
except Exception:
te = time.time()
tenant = args[1].context.tenant
update_error_count(api, method, te - te, tenant)
raise
return wrap
return wrapper
def update_count(api, method, ex_time, tenant=None):
log.debug("Updating count stats for %s, %s on object %s" % (api,
method,
v1.stats))
v1.stats.add_api_request(tenant, ex_time)
v1.stats.request_count += 1
def update_error_count(api, method, ex_time, tenant=None):
log.debug("Updating count stats for %s, %s on object %s" % (api,
method,
v1.stats))
v1.stats.add_api_error(tenant, ex_time)
v1.stats.error_count += 1
v1.stats.request_count += 1
def init_stats():
if not v1.stats:
v1.stats = StatisticsCollection()

2
muranoapi/cmd/api.py
View File

@ -26,6 +26,7 @@ root = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir, os.pardir)
if os.path.exists(os.path.join(root, 'muranoapi', '__init__.py')): if os.path.exists(os.path.join(root, 'muranoapi', '__init__.py')):
sys.path.insert(0, root) sys.path.insert(0, root)
from muranoapi.api.v1 import statistics
from muranoapi.common import config from muranoapi.common import config
from muranoapi.common import server from muranoapi.common import server
from muranoapi.common import statservice as stats from muranoapi.common import statservice as stats
@ -38,6 +39,7 @@ def main():
try: try:
config.parse_args() config.parse_args()
log.setup('muranoapi') log.setup('muranoapi')
statistics.init_stats()
launcher = service.ServiceLauncher() launcher = service.ServiceLauncher()

47
muranoapi/common/statservice.py
View File

@ -13,12 +13,20 @@
# under the License. # under the License.
import eventlet import eventlet
import json
import socket
import time
from muranoapi.api import v1
from muranoapi.api.v1 import statistics
from muranoapi.common import config from muranoapi.common import config
from muranoapi.db.services import stats as db_stats
from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import service from muranoapi.openstack.common import service
conf = config.CONF.stats conf = config.CONF.stats
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -26,6 +34,10 @@ log = logging.getLogger(__name__)
class StatsCollectingService(service.Service): class StatsCollectingService(service.Service):
def __init__(self): def __init__(self):
super(StatsCollectingService, self).__init__() super(StatsCollectingService, self).__init__()
statistics.init_stats()
self._hostname = socket.gethostname()
self._stats_db = db_stats.Statistics()
self._prev_time = time.time()
def start(self): def start(self):
super(StatsCollectingService, self).start() super(StatsCollectingService, self).start()
@ -42,3 +54,38 @@ class StatsCollectingService(service.Service):
def update_stats(self): def update_stats(self):
log.debug(_("Updating statistic information.")) log.debug(_("Updating statistic information."))
log.debug("Stats object: %s" % v1.stats)
log.debug("Stats: Requests:%s Errors: %s Ave.Res.Time %2.4f\n"
"Per tenant: %s" %
(v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant))
try:
stats = self._stats_db.get_stats_by_host(self._hostname)
if stats is None:
self._stats_db.create(self._hostname,
v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant)
return
now = time.time()
t_delta = now - self._prev_time
requests_per_second = (v1.stats.request_count -
stats.request_count) / t_delta
errors_per_second = (v1.stats.error_count -
stats.error_count) / t_delta
self._prev_time = now
stats.request_count = v1.stats.request_count
stats.error_count = v1.stats.error_count
stats.average_response_time = v1.stats.average_time
stats.requests_per_tenant = json.dumps(v1.stats.
requests_per_tenant)
stats.requests_per_second = requests_per_second
stats.errors_per_second = errors_per_second
self._stats_db.update(self._hostname, stats)
except Exception as e:
log.error(_("Failed to get statistics object "
"form a database. %s" % e))

41
muranoapi/db/migrate_repo/versions/003_add_stats_table.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table(
'apistats',
meta,
schema.Column('id', types.Integer(), primary_key=True),
schema.Column('host', types.String(80)),
schema.Column('request_count', types.BigInteger()),
schema.Column('error_count', types.BigInteger()),
schema.Column('average_response_time', types.Float()),
schema.Column('requests_per_tenant', types.Text()),
schema.Column('requests_per_second', types.Float()),
schema.Column('errors_per_second', types.Float()),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False))
stats.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table('apistats', meta, autoload=True)
stats.drop()

23
muranoapi/db/models.py
View File

@ -186,11 +186,29 @@ class Status(BASE, ModelBase):
return dictionary return dictionary
class ApiStats(BASE, ModelBase):
__tablename__ = 'apistats'
id = sa.Column(sa.Integer(), primary_key=True)
host = sa.Column(sa.String(80))
request_count = sa.Column(sa.BigInteger())
error_count = sa.Column(sa.BigInteger())
average_response_time = sa.Column(sa.Float())
requests_per_tenant = sa.Column(sa.Text())
requests_per_second = sa.Column(sa.Float())
errors_per_second = sa.Column(sa.Float())
def to_dict(self):
dictionary = super(ApiStats, self).to_dict()
return dictionary
def register_models(engine): def register_models(engine):
""" """
Creates database tables for all models with the given engine Creates database tables for all models with the given engine
""" """
models = (Environment, Status, Session, Deployment) models = (Environment, Status, Session, Deployment,
ApiStats)
for model in models: for model in models:
model.metadata.create_all(engine) model.metadata.create_all(engine)
@ -199,6 +217,7 @@ def unregister_models(engine):
""" """
Drops database tables for all models with the given engine Drops database tables for all models with the given engine
""" """
models = (Environment, Status, Session, Deployment) models = (Environment, Status, Session, Deployment,
ApiStats)
for model in models: for model in models:
model.metadata.drop_all(engine) model.metadata.drop_all(engine)

51
muranoapi/db/services/stats.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from muranoapi.db import models as m
from muranoapi.db import session as db_session
class Statistics(object):
@staticmethod
def get_stats_by_id(stats_id):
db = db_session.get_session()
stats = db.query(m.ApiStats).get(stats_id)
return stats
@staticmethod
def get_stats_by_host(host):
db = db_session.get_session()
stats = db.query(m.ApiStats).filter(m.ApiStats.host == host).first()
return stats
@staticmethod
def create(host, request_count, error_count,
average_response_time, request_per_tenant):
stats = m.ApiStats()
stats.host = host
stats.request_count = request_count
stats.error_count = error_count
stats.average_response_time = average_response_time
stats.request_per_tenant = request_per_tenant
stats.request_per_second = 0.0
stats.errors_per_second = 0.0
db = db_session.get_session()
stats.save(db)
@staticmethod
def update(host, stats):
db = db_session.get_session()
stats.save(db)