Merge "Optimize storage schema for Redis driver"

This commit is contained in:
Zuul 2019-05-20 09:36:26 +00:00 committed by Gerrit Code Review
commit 84f164d923
2 changed files with 42 additions and 6 deletions

View File

@ -47,7 +47,8 @@ class Redis(base.Driver):
# only connection over network is supported with schema
# redis://[:password]@host[:port][/db]
self.db = StrictRedis.from_url(self.connection_str)
self.namespace = "osprofiler:"
self.namespace_opt = "osprofiler_opt:"
self.namespace = "osprofiler:" # legacy
self.namespace_error = "osprofiler_error:"
@classmethod
@ -73,9 +74,8 @@ class Redis(base.Driver):
data = info.copy()
data["project"] = self.project
data["service"] = self.service
key = self.namespace + data["base_id"] + "_" + data["trace_id"] + "_" + \
data["timestamp"]
self.db.set(key, jsonutils.dumps(data))
key = self.namespace_opt + data["base_id"]
self.db.lpush(key, jsonutils.dumps(data))
if (self.filter_error_trace
and data.get("info", {}).get("etype") is not None):
@ -100,6 +100,19 @@ class Redis(base.Driver):
"""
fields = set(fields or self.default_trace_fields)
# first get legacy events
result = self._list_traces_legacy(fields)
# with optimized schema trace events are stored in a list
ids = self.db.scan_iter(match=self.namespace_opt + "*")
for i in ids:
# for each trace query the first event to have a timestamp
first_event = jsonutils.loads(self.db.lindex(i, 1))
result.append({key: value for key, value in first_event.items()
if key in fields})
return result
def _list_traces_legacy(self, fields):
# With current schema every event is stored under its own unique key
# To query all traces we first need to get all keys, then
# get all events, sort them and pick up only the first one
@ -134,8 +147,15 @@ class Redis(base.Driver):
:param base_id: Base id of trace elements.
"""
for key in self.db.scan_iter(match=self.namespace + base_id + "*"):
data = self.db.get(key)
def iterate_events():
for key in self.db.scan_iter(
match=self.namespace + base_id + "*"): # legacy
yield self.db.get(key)
for event in self.db.lrange(self.namespace_opt + base_id, 0, -1):
yield event
for data in iterate_events():
n = jsonutils.loads(data)
trace_id = n["trace_id"]
parent_id = n["parent_id"]

View File

@ -0,0 +1,16 @@
---
features:
- |
Redis storage schema is optimized for higher performance.
Previously Redis driver stored each tracing event under its own key,
as result both list and get operations required full scan of the database.
With the optimized schema traces are stored as Redis lists under a key
equal to trace id. So list operation iterates only over unique
trace ids and get operation retrieves content of a specified list.
Note that list operation still needs to retrieve at least 1 event
from the trace to get a timestamp.
upgrade:
- |
The optimized Redis driver is backward compatible: while new events are stored
using new schema the driver can retrieve existing events using both old and new
schemas.