Support for Opentelemetry Tracing. Programmatic instrumentation of Fl…

…ask, SQLAlchemy, and Requests libraries (#118)

Signed-off-by: Mike Kingsbury <[email protected]>

pyproject.toml

@@ -51,6 +51,13 @@ Flask-Session = "^0.5.0"
 # Flask-Session seems to be incompatible with werkzeug-3.0.0
 werkzeug = "<3.0.0"
 
+# tracing support
+opentelemetry-exporter-otlp = "^1.21.0"
+opentelemetry-instrumentation = "^0.42b0"
+opentelemetry-instrumentation-flask = "^0.42b0"
+opentelemetry-instrumentation-sqlalchemy = "^0.42b0"
+opentelemetry-instrumentation-requests = "^0.42b0"
+
 [tool.poetry.extras]
 test = [
     "pytest",

resultsdb/__init__.py

@@ -40,6 +40,7 @@
 from resultsdb.controllers.api_v3 import api as api_v3, create_endpoints
 from resultsdb.messaging import load_messaging_plugin
 from resultsdb.models import db
+from resultsdb.tracing import setup_tracing
 from . import config
 
 
@@ -95,6 +96,8 @@ def create_app(config_obj=None):
 
     setup_logging(app)
 
+    setup_tracing(app)
+
     app.logger.info("Using configuration object: %s", config_obj)
     if openshift:
         app.logger.info("Using OpenShift configuration")

resultsdb/config.py

@@ -115,6 +115,8 @@ class Config(object):
 
     # Publish Taskotron-compatible fedmsgs on the 'taskotron' topic
     MESSAGE_BUS_PUBLISH_TASKOTRON = False
+    OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = None
+    OTEL_EXPORTER_SERVICE_NAME = "resultsdb"
 
 
 class ProductionConfig(Config):

resultsdb/messaging.py

@@ -37,6 +37,8 @@
     ConnectionException,
 )
 
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
 log = logging.getLogger(__name__)
 
 SERIALIZE = Serializer().serialize
@@ -152,6 +154,9 @@ class DummyPlugin(MessagingPlugin):
     history = []
 
     def publish(self, message):
+        # Add telemetry information. This includes an extra key
+        # traceparent.
+        TraceContextTextMapPropagator().inject(message)
         self.history.append(message)
         log.info("%r->%r" % (self, message))
 
@@ -209,6 +214,10 @@ def __init__(self, **kwargs):
 
     def publish(self, msg):
         msg = json.dumps(msg)
+        # Add telemetry information. This includes an extra key
+        # traceparent.
+        TraceContextTextMapPropagator().inject(msg)
+
         kwargs = dict(body=msg, headers={}, destination=self.destination)
 
         conn = stomp.connect.StompConnection11(**self.connection)

resultsdb/tracing.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+
+def setup_tracing(app):  # pragma: no cover
+    endpoint = app.config.get("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
+    service_name = app.config.get("OTEL_EXPORTER_SERVICE_NAME")
+    if not endpoint or not service_name:
+        return
+    resource = Resource(attributes={SERVICE_NAME: service_name})
+    provider = TracerProvider(resource=resource)
+    instrumentor = FlaskInstrumentor()
+    instrumentor.instrument_app(app)
+    SQLAlchemyInstrumentor().instrument(
+        enable_commenter=True,
+        commenter_options={
+            "db_driver": True,
+        },
+    )
+    RequestsInstrumentor().instrument()
+    otlp_exporter = OTLPSpanExporter(endpoint=app.config["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"])
+    processor = BatchSpanProcessor(otlp_exporter)
+    # processor = BatchSpanProcessor(ConsoleSpanExporter())
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)