openstack/murano
Code Issues Proposed changes

Merge "Add per API call statistics"

Browse Source
This commit is contained in:
Jenkins 2014-03-14 15:25:38 +00:00 committed by Gerrit Code Review
commit 2b0c534a0e
11 changed files with 287 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
muranoapi/api/v1/__init__.py
View File

@ -15,6 +15,8 @@
from muranoapi.db import models
from muranoapi.db import session as db_session
stats = None
def get_draft(environment_id=None, session_id=None):
unit = db_session.get_session()

8
muranoapi/api/v1/deployments.py
View File

@ -11,13 +11,15 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import desc
from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.common import utils
from muranoapi.db import models
from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi
@ -25,8 +27,11 @@ from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__)
API_NAME = 'Deployments'
class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
def index(self, request, environment_id):
unit = db_session.get_session()
verify_and_get_env(unit, environment_id, request)
@ -38,6 +43,7 @@ class Controller(object):
in result]
return {'deployments': deployments}
@statistics.stats_count(API_NAME, 'Statuses')
def statuses(self, request, environment_id, deployment_id):
unit = db_session.get_session()
query = unit.query(models.Status) \

12
muranoapi/api/v1/environments.py
View File

@ -15,20 +15,27 @@
from sqlalchemy import desc
from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.common import utils
from muranoapi.db import models
from muranoapi.db.services import core_services
from muranoapi.db.services import environments as envs
from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__)
API_NAME = 'Environments'
class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
def index(self, request):
log.debug(_('Environments:List'))
@ -39,6 +46,7 @@ class Controller(object):
return {"environments": environments}
@statistics.stats_count(API_NAME, 'Create')
def create(self, request, body):
log.debug(_('Environments:Create <Body {0}>'.format(body)))
@ -47,6 +55,7 @@ class Controller(object):
return environment.to_dict()
@statistics.stats_count(API_NAME, 'Show')
def show(self, request, environment_id):
log.debug(_('Environments:Show <Id: {0}>'.format(environment_id)))
@ -76,6 +85,7 @@ class Controller(object):
return env
@statistics.stats_count(API_NAME, 'Update')
def update(self, request, environment_id, body):
log.debug(_('Environments:Update <Id: {0}, '
'Body: {1}>'.format(environment_id, body)))
@ -98,6 +108,7 @@ class Controller(object):
return environment.to_dict()
@statistics.stats_count(API_NAME, 'Delete')
def delete(self, request, environment_id):
log.debug(_('Environments:Delete <Id: {0}>'.format(environment_id)))
@ -117,6 +128,7 @@ class Controller(object):
envs.EnvironmentServices.delete(environment_id,
request.context.auth_token)
@statistics.stats_count(API_NAME, 'LastStatus')
def last(self, request, environment_id):
session_id = None
if hasattr(request, 'context') and request.context.session:

9
muranoapi/api/v1/services.py
View File

@ -15,7 +15,10 @@ import functools as func
from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.db.services import core_services
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi
@ -24,6 +27,8 @@ from muranocommon.helpers import token_sanitizer
log = logging.getLogger(__name__)
API_NAME = 'Services'
def normalize_path(f):
@func.wraps(f)
@ -39,6 +44,7 @@ def normalize_path(f):
class Controller(object):
@statistics.stats_count(API_NAME, 'Index')
@utils.verify_env
@normalize_path
def get(self, request, environment_id, path):
@ -57,6 +63,7 @@ class Controller(object):
raise exc.HTTPNotFound
return result
@statistics.stats_count(API_NAME, 'Create')
@utils.verify_session
@utils.verify_env
@normalize_path
@ -73,6 +80,7 @@ class Controller(object):
raise exc.HTTPNotFound
return result
@statistics.stats_count(API_NAME, 'Update')
@utils.verify_session
@utils.verify_env
@normalize_path
@ -89,6 +97,7 @@ class Controller(object):
raise exc.HTTPNotFound
return result
@statistics.stats_count(API_NAME, 'Delete')
@utils.verify_session
@utils.verify_env
@normalize_path

7
muranoapi/api/v1/sessions.py
View File

@ -14,19 +14,23 @@
from webob import exc
from muranoapi.api.v1 import statistics
from muranoapi.db import models
from muranoapi.db.services import environments as envs
from muranoapi.db.services import sessions
from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi
log = logging.getLogger(__name__)
API_NAME = 'Sessions'
class Controller(object):
@statistics.stats_count(API_NAME, 'Create')
def configure(self, request, environment_id):
log.debug(_('Session:Configure <EnvId: {0}>'.format(environment_id)))
@ -56,6 +60,7 @@ class Controller(object):
return session.to_dict()
@statistics.stats_count(API_NAME, 'Index')
def show(self, request, environment_id, session_id):
log.debug(_('Session:Show <SessionId: {0}>'.format(session_id)))
@ -85,6 +90,7 @@ class Controller(object):
return session.to_dict()
@statistics.stats_count(API_NAME, 'Delete')
def delete(self, request, environment_id, session_id):
log.debug(_('Session:Delete <SessionId: {0}>'.format(session_id)))
@ -117,6 +123,7 @@ class Controller(object):
return None
@statistics.stats_count(API_NAME, 'Deploy')
def deploy(self, request, environment_id, session_id):
log.debug(_('Session:Deploy <SessionId: {0}>'.format(session_id)))

88
muranoapi/api/v1/statistics.py Normal file
View File

@ -0,0 +1,88 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from muranoapi.api import v1
from muranoapi.openstack.common import log as logging
log = logging.getLogger(__name__)
class StatisticsCollection(object):
request_count = 0
error_count = 0
average_time = 0.0
requests_per_tenant = {}
errors_per_tenant = {}
def add_api_request(self, tenant, ex_time):
self.average_time = (self.average_time * self.request_count +
ex_time) / (self.request_count + 1)
if tenant:
tenant_count = self.requests_per_tenant.get(tenant, 0)
tenant_count += 1
self.requests_per_tenant[tenant] = tenant_count
def add_api_error(self, tenant, ex_time):
self.average_time = (self.average_time * self.request_count +
ex_time) / (self.request_count + 1)
if tenant:
tenant_count = self.errors_per_tenant.get(tenant, 0)
tenant_count += 1
self.errors_per_tenant[tenant] = tenant_count
def stats_count(api, method):
def wrapper(func):
def wrap(*args, **kwargs):
try:
ts = time.time()
result = func(*args, **kwargs)
te = time.time()
tenant = args[1].context.tenant
update_count(api, method, te - ts,
tenant)
return result
except Exception:
te = time.time()
tenant = args[1].context.tenant
update_error_count(api, method, te - te, tenant)
raise
return wrap
return wrapper
def update_count(api, method, ex_time, tenant=None):
log.debug("Updating count stats for %s, %s on object %s" % (api,
method,
v1.stats))
v1.stats.add_api_request(tenant, ex_time)
v1.stats.request_count += 1
def update_error_count(api, method, ex_time, tenant=None):
log.debug("Updating count stats for %s, %s on object %s" % (api,
method,
v1.stats))
v1.stats.add_api_error(tenant, ex_time)
v1.stats.error_count += 1
v1.stats.request_count += 1
def init_stats():
if not v1.stats:
v1.stats = StatisticsCollection()

2
muranoapi/cmd/api.py
View File

@ -26,6 +26,7 @@ root = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir, os.pardir)
if os.path.exists(os.path.join(root, 'muranoapi', '__init__.py')):
sys.path.insert(0, root)
from muranoapi.api.v1 import statistics
from muranoapi.common import config
from muranoapi.common import server
from muranoapi.common import statservice as stats
@ -38,6 +39,7 @@ def main():
try:
config.parse_args()
log.setup('muranoapi')
statistics.init_stats()
launcher = service.ServiceLauncher()

47
muranoapi/common/statservice.py
View File

@ -13,12 +13,20 @@
# under the License.
import eventlet
import json
import socket
import time
from muranoapi.api import v1
from muranoapi.api.v1 import statistics
from muranoapi.common import config
from muranoapi.db.services import stats as db_stats
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import service
conf = config.CONF.stats
log = logging.getLogger(__name__)
@ -26,6 +34,10 @@ log = logging.getLogger(__name__)
class StatsCollectingService(service.Service):
def __init__(self):
super(StatsCollectingService, self).__init__()
statistics.init_stats()
self._hostname = socket.gethostname()
self._stats_db = db_stats.Statistics()
self._prev_time = time.time()
def start(self):
super(StatsCollectingService, self).start()
@ -42,3 +54,38 @@ class StatsCollectingService(service.Service):
def update_stats(self):
log.debug(_("Updating statistic information."))
log.debug("Stats object: %s" % v1.stats)
log.debug("Stats: Requests:%s Errors: %s Ave.Res.Time %2.4f\n"
"Per tenant: %s" %
(v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant))
try:
stats = self._stats_db.get_stats_by_host(self._hostname)
if stats is None:
self._stats_db.create(self._hostname,
v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant)
return
now = time.time()
t_delta = now - self._prev_time
requests_per_second = (v1.stats.request_count -
stats.request_count) / t_delta
errors_per_second = (v1.stats.error_count -
stats.error_count) / t_delta
self._prev_time = now
stats.request_count = v1.stats.request_count
stats.error_count = v1.stats.error_count
stats.average_response_time = v1.stats.average_time
stats.requests_per_tenant = json.dumps(v1.stats.
requests_per_tenant)
stats.requests_per_second = requests_per_second
stats.errors_per_second = errors_per_second
self._stats_db.update(self._hostname, stats)
except Exception as e:
log.error(_("Failed to get statistics object "
"form a database. %s" % e))

41
muranoapi/db/migrate_repo/versions/003_add_stats_table.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table(
'apistats',
meta,
schema.Column('id', types.Integer(), primary_key=True),
schema.Column('host', types.String(80)),
schema.Column('request_count', types.BigInteger()),
schema.Column('error_count', types.BigInteger()),
schema.Column('average_response_time', types.Float()),
schema.Column('requests_per_tenant', types.Text()),
schema.Column('requests_per_second', types.Float()),
schema.Column('errors_per_second', types.Float()),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False))
stats.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table('apistats', meta, autoload=True)
stats.drop()

23
muranoapi/db/models.py
View File

@ -186,11 +186,29 @@ class Status(BASE, ModelBase):
return dictionary
class ApiStats(BASE, ModelBase):
__tablename__ = 'apistats'
id = sa.Column(sa.Integer(), primary_key=True)
host = sa.Column(sa.String(80))
request_count = sa.Column(sa.BigInteger())
error_count = sa.Column(sa.BigInteger())
average_response_time = sa.Column(sa.Float())
requests_per_tenant = sa.Column(sa.Text())
requests_per_second = sa.Column(sa.Float())
errors_per_second = sa.Column(sa.Float())
def to_dict(self):
dictionary = super(ApiStats, self).to_dict()
return dictionary
def register_models(engine):
"""
Creates database tables for all models with the given engine
"""
models = (Environment, Status, Session, Deployment)
models = (Environment, Status, Session, Deployment,
ApiStats)
for model in models:
model.metadata.create_all(engine)
@ -199,6 +217,7 @@ def unregister_models(engine):
"""
Drops database tables for all models with the given engine
"""
models = (Environment, Status, Session, Deployment)
models = (Environment, Status, Session, Deployment,
ApiStats)
for model in models:
model.metadata.drop_all(engine)

51
muranoapi/db/services/stats.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from muranoapi.db import models as m
from muranoapi.db import session as db_session
class Statistics(object):
@staticmethod
def get_stats_by_id(stats_id):
db = db_session.get_session()
stats = db.query(m.ApiStats).get(stats_id)
return stats
@staticmethod
def get_stats_by_host(host):
db = db_session.get_session()
stats = db.query(m.ApiStats).filter(m.ApiStats.host == host).first()
return stats
@staticmethod
def create(host, request_count, error_count,
average_response_time, request_per_tenant):
stats = m.ApiStats()
stats.host = host
stats.request_count = request_count
stats.error_count = error_count
stats.average_response_time = average_response_time
stats.request_per_tenant = request_per_tenant
stats.request_per_second = 0.0
stats.errors_per_second = 0.0
db = db_session.get_session()
stats.save(db)
@staticmethod
def update(host, stats):
db = db_session.get_session()
stats.save(db)