openstack/ceilometer
Code Issues Proposed changes

[MongoDB] Refactor indexes for meter and resources

Browse Source 
Currently Ceilometer uses redundant indexes in MongoDB and in same
time doesn't cover get_resources sorting by indexes.
In this CR source field indexing is deleted and resource.project_id,
resource.user_id indexes are combined to one.
New compound index covers the get_resource sorting and its parts
are used in queries by timestamp, project_id, user_id.

Change-Id: I077317533fd30865eea8a3f32e3c3a7f53fa6343
Closes: bug 1433924
This commit is contained in:
Ilya Tyaptin 2015-07-30 14:18:42 +03:00
parent 29d3cd405b
commit 4b8929206a
2 changed files with 29 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ceilometer/storage/impl_mongodb.py
View File

@ -207,28 +207,30 @@ class Connection(pymongo_base.Connection):
name_qualifier = dict(user_id='', project_id='project_')
background = dict(user_id=False, project_id=True)
for primary in ['user_id', 'project_id']:
name = 'resource_%sidx' % name_qualifier[primary]
self.db.resource.create_index([
(primary, pymongo.ASCENDING),
('source', pymongo.ASCENDING),
], name=name, background=background[primary])
name = 'meter_%sidx' % name_qualifier[primary]
self.db.meter.create_index([
('resource_id', pymongo.ASCENDING),
(primary, pymongo.ASCENDING),
('counter_name', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
('source', pymongo.ASCENDING),
], name=name, background=background[primary])
self.db.resource.create_index([('last_sample_timestamp',
pymongo.DESCENDING)],
name='last_sample_timestamp_idx',
sparse=True)
self.db.meter.create_index([('timestamp', pymongo.DESCENDING)],
name='timestamp_idx')
# NOTE(ityaptin) This index covers get_resource requests sorting
# and MongoDB uses part of this compound index for different
# queries based on any of user_id, project_id, last_sample_timestamp
# fields
self.db.resource.create_index([('user_id', pymongo.DESCENDING),
('project_id', pymongo.DESCENDING),
('last_sample_timestamp',
pymongo.DESCENDING)],
name='resource_user_project_timestamp',)
self.db.resource.create_index([('last_sample_timestamp',
pymongo.DESCENDING)],
name='last_sample_timestamp_idx')
# update or create time_to_live index
ttl = cfg.CONF.database.metering_time_to_live
self.update_ttl(ttl, 'meter_ttl', 'timestamp', self.db.meter)

16
ceilometer/storage/mongo/utils.py
View File

@ -26,11 +26,14 @@ from oslo_config import cfg
from oslo_log import log
from oslo_utils import netutils
import pymongo
import pymongo.errors
import six
from six.moves.urllib import parse
from ceilometer.i18n import _
ERROR_INDEX_WITH_DIFFERENT_SPEC_ALREADY_EXISTS = 86
LOG = log.getLogger(__name__)
# FIXME(dhellmann): Configuration options are not part of the Oslo
@ -455,6 +458,19 @@ class MongoProxy(object):
# exception.
return CursorProxy(self.conn.find(*args, **kwargs))
def create_index(self, keys, name=None, *args, **kwargs):
try:
self.conn.create_index(keys, name=name, *args, **kwargs)
except pymongo.errors.OperationFailure as e:
if e.code is ERROR_INDEX_WITH_DIFFERENT_SPEC_ALREADY_EXISTS:
LOG.info(_("Index %s will be recreate.") % name)
self._recreate_index(keys, name, *args, **kwargs)
@safe_mongo_call
def _recreate_index(self, keys, name, *args, **kwargs):
self.conn.drop_index(name)
self.conn.create_index(keys, name=name, *args, **kwargs)
def __getattr__(self, item):
"""Wrap MongoDB connection.