From ada8b751e1970d2b5206e1b0e6da40bcb11787c9 Mon Sep 17 00:00:00 2001 From: Sahid Orentino Ferdjaoui Date: Tue, 14 Mar 2023 12:22:47 +0100 Subject: [PATCH] add support of jaeger+otlp for jaeger tracing Signed-off-by: Sahid Orentino Ferdjaoui Change-Id: I74cdcb2aa99b0162ba1c14059111f09d0bb534e3 --- devstack/lib/osprofiler | 15 +- lower-constraints.txt | 2 + osprofiler/_utils.py | 12 ++ osprofiler/drivers/__init__.py | 1 + osprofiler/drivers/jaeger_otlp.py | 131 ++++++++++++++++++ .../tests/unit/drivers/test_jaeger_otlp.py | 84 +++++++++++ ...r-for-jaeger-tracing-cb932038ad580ac2.yaml | 8 ++ test-requirements.txt | 2 + 8 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 osprofiler/drivers/jaeger_otlp.py create mode 100644 osprofiler/tests/unit/drivers/test_jaeger_otlp.py create mode 100644 releasenotes/notes/otlp-driver-for-jaeger-tracing-cb932038ad580ac2.yaml diff --git a/devstack/lib/osprofiler b/devstack/lib/osprofiler index 5176a1e..b8e3634 100644 --- a/devstack/lib/osprofiler +++ b/devstack/lib/osprofiler @@ -58,19 +58,27 @@ function install_redis() { pip_install_gr redis } -function install_jaeger() { +function install_jaeger_backend() { if is_ubuntu; then install_package docker.io start_service docker add_user_to_group $STACK_USER docker - sg docker -c "docker run -d --name jaeger -p 6831:6831/udp -p 16686:16686 jaegertracing/all-in-one:1.7" + sg docker -c "docker run -d --name jaeger -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -e COLLECTOR_OTLP_ENABLED=true -p 6831:6831/udp -p 6832:6832/udp -p 5778:5778 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 14250:14250 -p 14268:14268 -p 14269:14269 -p 9411:9411 jaegertracing/all-in-one:1.42" else exit_distro_not_supported "docker.io installation" fi +} +function install_jaeger() { + install_jaeger_backend pip_install jaeger-client } +function install_jaeger_otlp() { + install_jaeger_backend + pip_install opentelemetry-sdk opentelemetry-exporter-otlp +} + function install_elasticsearch() { if is_ubuntu; then install_package docker.io @@ -108,6 +116,9 @@ function install_osprofiler_collector() { elif [ "$OSPROFILER_COLLECTOR" == "jaeger" ]; then install_jaeger OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"jaeger://localhost:6831"} + elif [ "$OSPROFILER_COLLECTOR" == "jaeger+otlp" ]; then + install_jaeger_otlp + OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"jaeger+otlp://localhost:4318"} elif [ "$OSPROFILER_COLLECTOR" == "elasticsearch" ]; then install_elasticsearch OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"elasticsearch://elastic:changeme@localhost:9200"} diff --git a/lower-constraints.txt b/lower-constraints.txt index aafaad0..91c4dcd 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -5,6 +5,8 @@ dulwich===0.15.0 elasticsearch===2.0.0 importlib_metadata==1.7.0 jaeger-client==3.8.0 +opentelemetry-exporter-otlp==1.16.0 +opentelemetry-sdk==1.16.0 netaddr===0.7.18 openstackdocstheme==2.2.1 oslo.concurrency===3.26.0 diff --git a/osprofiler/_utils.py b/osprofiler/_utils.py index 7563fb5..d903b9b 100644 --- a/osprofiler/_utils.py +++ b/osprofiler/_utils.py @@ -161,3 +161,15 @@ def shorten_id(span_id): # Return a new short id for this short_id = shorten_id(uuidutils.generate_uuid()) return short_id + + +def uuid_to_int128(span_uuid): + """Convert from uuid4 to 128 bit id for OpenTracing""" + if isinstance(span_uuid, int): + return span_uuid + try: + span_int = uuid.UUID(span_uuid).int + except ValueError: + # Return a new short id for this + span_int = uuid_to_int128(uuidutils.generate_uuid()) + return span_int diff --git a/osprofiler/drivers/__init__.py b/osprofiler/drivers/__init__.py index 022b094..f056a5c 100644 --- a/osprofiler/drivers/__init__.py +++ b/osprofiler/drivers/__init__.py @@ -1,6 +1,7 @@ from osprofiler.drivers import base # noqa from osprofiler.drivers import elasticsearch_driver # noqa from osprofiler.drivers import jaeger # noqa +from osprofiler.drivers import jaeger_otlp # noqa from osprofiler.drivers import loginsight # noqa from osprofiler.drivers import messaging # noqa from osprofiler.drivers import mongodb # noqa diff --git a/osprofiler/drivers/jaeger_otlp.py b/osprofiler/drivers/jaeger_otlp.py new file mode 100644 index 0000000..3086f0d --- /dev/null +++ b/osprofiler/drivers/jaeger_otlp.py @@ -0,0 +1,131 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +from urllib import parse as parser + +from oslo_config import cfg + +from osprofiler import _utils as utils +from osprofiler.drivers import jaeger +from osprofiler import exc + + +class JaegerOTLP(jaeger.Jaeger): + def __init__(self, connection_str, project=None, service=None, host=None, + conf=cfg.CONF, **kwargs): + """Jaeger driver using OTLP exporters driver for OSProfiler.""" + try: + from opentelemetry import trace as trace_api + + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter # noqa + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + self.trace_api = trace_api + except ImportError: + raise exc.CommandError( + "To use OSProfiler with Jaeger OTLP expoerts, " + "please install `opentelemetry-sdk` and " + "opentelemetry-exporter-otlp libraries. " + "To install with pip:\n `pip install opentelemetry-sdk " + "opentelemetry-exporter-otlp`.") + + + service_name = self._get_service_name(conf, project, service) + resource = Resource(attributes={ + "service.name": service_name + }) + + parsed_url = parser.urlparse(connection_str) + # TODO("sahid"): We also want to handle https scheme? + parsed_url = parsed_url._replace(scheme="http") + + self.trace_api.set_tracer_provider( + TracerProvider(resource=resource)) + self.tracer = self.trace_api.get_tracer(__name__) + + exporter = OTLPSpanExporter("{}/v1/traces".format( + parsed_url.geturl())) + self.trace_api.get_tracer_provider().add_span_processor( + BatchSpanProcessor(exporter)) + + self.spans = collections.deque() + + @classmethod + def get_name(cls): + return "jaeger+otlp" + + def _kind(self, name): + if "wsgi" in name: + return self.trace_api.SpanKind.SERVER + elif ("db" in name or "http_client" in name or "api" in name): + return self.trace_api.SpanKind.CLIENT + return self.trace_api.SpanKind.INTERNAL + + def _name(self, payload): + info = payload["info"] + if info.get("request"): + return "{}_{}".format( + info["request"]["method"], info["request"]["path"]) + elif info.get("db"): + return "SQL_{}".format( + info["db"]["statement"].split(' ', 1)[0].upper()) + elif info.get("requests"): + return info["requests"]["method"] + return payload["name"].rstrip("-start") + + def notify(self, payload): + if payload["name"].endswith("start"): + parent = self.trace_api.SpanContext( + trace_id=utils.uuid_to_int128(payload["base_id"]), + span_id=utils.shorten_id(payload["parent_id"]), + is_remote=False, + trace_flags=self.trace_api.TraceFlags( + self.trace_api.TraceFlags.SAMPLED)) + + ctx = self.trace_api.set_span_in_context( + self.trace_api.NonRecordingSpan(parent)) + + # OTLP Tracing span + span = self.tracer.start_span( + name=self._name(payload), + kind=self._kind(payload['name']), + attributes=self.create_span_tags(payload), + context=ctx) + + span._context = self.trace_api.SpanContext( + trace_id=span.context.trace_id, + span_id=utils.shorten_id(payload["trace_id"]), + is_remote=span.context.is_remote, + trace_flags=span.context.trace_flags, + trace_state=span.context.trace_state) + + self.spans.append(span) + else: + span = self.spans.pop() + + # Store result of db call and function call + for call in ("db", "function"): + if payload.get("info", {}).get(call): + span.set_attribute( + "result", payload["info"][call]["result"]) + # Span error tag and log + if payload["info"].get("etype"): + span.set_attribute("error", True) + span.add_event("log", { + "error.kind": payload["info"]["etype"], + "message": payload["info"]["message"]}) + span.end() diff --git a/osprofiler/tests/unit/drivers/test_jaeger_otlp.py b/osprofiler/tests/unit/drivers/test_jaeger_otlp.py new file mode 100644 index 0000000..c025958 --- /dev/null +++ b/osprofiler/tests/unit/drivers/test_jaeger_otlp.py @@ -0,0 +1,84 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from oslo_config import cfg + +from osprofiler.drivers import jaeger_otlp +from osprofiler import opts +from osprofiler.tests import test + + +class JaegerOTLPTestCase(test.TestCase): + + def setUp(self): + super(JaegerOTLPTestCase, self).setUp() + + opts.set_defaults(cfg.CONF) + + self.payload_start = { + "name": "api-start", + "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee", + "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1", + "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b", + "timestamp": "2018-05-03T04:31:51.781381", + "info": { + "host": "test" + } + } + + self.payload_stop = { + "name": "api-stop", + "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee", + "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1", + "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b", + "timestamp": "2018-05-03T04:31:51.781381", + "info": { + "host": "test", + "function": { + "result": 1 + } + } + } + + self.driver = jaeger_otlp.JaegerOTLP( + "jaeger+otlp://127.0.0.1:6831", + project="nova", service="api", + conf=cfg.CONF) + + def test_notify_start(self): + self.driver.notify(self.payload_start) + self.assertEqual(1, len(self.driver.spans)) + + def test_notify_stop(self): + mock_end = mock.MagicMock() + self.driver.notify(self.payload_start) + self.driver.spans[0].end = mock_end + self.driver.notify(self.payload_stop) + mock_end.assert_called_once() + + def test_service_name_default(self): + self.assertEqual("pr1-svc1", self.driver._get_service_name( + cfg.CONF, "pr1", "svc1")) + + def test_service_name_prefix(self): + cfg.CONF.set_default( + "service_name_prefix", "prx1", "profiler_jaeger") + self.assertEqual("prx1-pr1-svc1", self.driver._get_service_name( + cfg.CONF, "pr1", "svc1")) + + def test_process_tags(self): + # Need to be implemented. + pass diff --git a/releasenotes/notes/otlp-driver-for-jaeger-tracing-cb932038ad580ac2.yaml b/releasenotes/notes/otlp-driver-for-jaeger-tracing-cb932038ad580ac2.yaml new file mode 100644 index 0000000..90d0e68 --- /dev/null +++ b/releasenotes/notes/otlp-driver-for-jaeger-tracing-cb932038ad580ac2.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + An OTLP (OpenTelemetry) exporter for Jaeger tracing is now supported. The + current support is experimental but the aim is to deprecate and + remove legacy Jaeger driver which is using the already deprecated + python library jaeger client. Operators who want to use it should + enable `jaeger+otlp`. diff --git a/test-requirements.txt b/test-requirements.txt index 3e1bbd8..096429d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,5 +20,7 @@ redis>=2.10.0 # MIT # For Jaeger Tracing jaeger-client>=3.8.0 # Apache-2.0 +opentelemetry-exporter-otlp>=1.16.0 +opentelemetry-sdk>=1.16.0 pre-commit>=2.6.0 # MIT