Add opentelemetry

docker-compose.prod.yml

@@ -15,13 +15,18 @@ services:
       LETSENCRYPT_HOST: openpredict.semanticscience.org
       VIRTUAL_PORT: 8808
 
+      JAEGER_ENABLED: true
+      JAEGER_HOST: jaeger-otel-agent.sri
+      JAEGER_PORT: 6831
+      OTEL_SERVICE_NAME: OPENPREDICT
+
       # OPENPREDICT_APIKEY: ${OPENPREDICT_APIKEY}
 
   jaeger:
     image: jaegertracing/all-in-one:latest
     # ports:
     #   - "16686:16686"
     #   - "4318:4318"
-    #   - "6831:6831"
+    #   - "6831:6831/udp"
     environment:
       - LOG_LEVEL=debug

src/trapi/main.py

@@ -110,3 +110,46 @@
 
 app.include_router(openpredict_api)
 app.include_router(drkg_model_api)
+
+
+def configure_otel(service_name, app):
+    # open telemetry https://github.com/ranking-agent/aragorn/blob/main/src/otel_config.py#L4
+    # https://ncatstranslator.github.io/TranslatorTechnicalDocumentation/deployment-guide/monitoring/
+    # https://github.com/TranslatorSRI/Jaeger-demo
+    if os.environ.get('JAEGER_ENABLED') == "True":
+        logging.info("starting up jaeger telemetry")
+        import warnings
+        from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+        from opentelemetry import trace
+        from opentelemetry.exporter.jaeger.thrift import JaegerExporter
+        from opentelemetry.sdk.resources import SERVICE_NAME as telemetery_service_name_key, Resource
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+
+        # httpx connections need to be open a little longer by the otel decorators
+        # but some libs display warnings of resource being unclosed.
+        # these supresses such warnings.
+        logging.captureWarnings(capture=True)
+        warnings.filterwarnings("ignore",category=ResourceWarning)
+        trace.set_tracer_provider(
+            TracerProvider(
+                resource=Resource.create({telemetery_service_name_key: service_name})
+            )
+        )
+        jaeger_host = os.environ.get('JAEGER_HOST', 'jaeger-otel-agent.sri')
+        jaeger_port = int(os.environ.get('JAEGER_PORT', '6831'))
+        jaeger_exporter = JaegerExporter(
+            agent_host_name=jaeger_host,
+            agent_port=jaeger_port,
+        )
+        trace.get_tracer_provider().add_span_processor(
+            BatchSpanProcessor(jaeger_exporter)
+        )
+        # tracer = trace.get_tracer(__name__)
+        FastAPIInstrumentor.instrument_app(app, tracer_provider=trace, excluded_urls="docs,openapi.json")
+        HTTPXClientInstrumentor().instrument()
+
+# Configure open telemetry if enabled
+service_name = os.environ.get('OTEL_SERVICE_NAME', 'OPENPREDICT')
+configure_otel(service_name, app)