diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 58f3c84..278fe9a 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,6 +15,11 @@ services: LETSENCRYPT_HOST: openpredict.semanticscience.org VIRTUAL_PORT: 8808 + JAEGER_ENABLED: true + JAEGER_HOST: jaeger-otel-agent.sri + JAEGER_PORT: 6831 + OTEL_SERVICE_NAME: OPENPREDICT + # OPENPREDICT_APIKEY: ${OPENPREDICT_APIKEY} jaeger: @@ -22,6 +27,6 @@ services: # ports: # - "16686:16686" # - "4318:4318" - # - "6831:6831" + # - "6831:6831/udp" environment: - LOG_LEVEL=debug \ No newline at end of file diff --git a/src/trapi/main.py b/src/trapi/main.py index e40fe19..6c11749 100644 --- a/src/trapi/main.py +++ b/src/trapi/main.py @@ -110,3 +110,46 @@ app.include_router(openpredict_api) app.include_router(drkg_model_api) + + +def configure_otel(service_name, app): + # open telemetry https://github.com/ranking-agent/aragorn/blob/main/src/otel_config.py#L4 + # https://ncatstranslator.github.io/TranslatorTechnicalDocumentation/deployment-guide/monitoring/ + # https://github.com/TranslatorSRI/Jaeger-demo + if os.environ.get('JAEGER_ENABLED') == "True": + logging.info("starting up jaeger telemetry") + import warnings + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + from opentelemetry import trace + from opentelemetry.exporter.jaeger.thrift import JaegerExporter + from opentelemetry.sdk.resources import SERVICE_NAME as telemetery_service_name_key, Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor + + # httpx connections need to be open a little longer by the otel decorators + # but some libs display warnings of resource being unclosed. + # these supresses such warnings. + logging.captureWarnings(capture=True) + warnings.filterwarnings("ignore",category=ResourceWarning) + trace.set_tracer_provider( + TracerProvider( + resource=Resource.create({telemetery_service_name_key: service_name}) + ) + ) + jaeger_host = os.environ.get('JAEGER_HOST', 'jaeger-otel-agent.sri') + jaeger_port = int(os.environ.get('JAEGER_PORT', '6831')) + jaeger_exporter = JaegerExporter( + agent_host_name=jaeger_host, + agent_port=jaeger_port, + ) + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(jaeger_exporter) + ) + # tracer = trace.get_tracer(__name__) + FastAPIInstrumentor.instrument_app(app, tracer_provider=trace, excluded_urls="docs,openapi.json") + HTTPXClientInstrumentor().instrument() + +# Configure open telemetry if enabled +service_name = os.environ.get('OTEL_SERVICE_NAME', 'OPENPREDICT') +configure_otel(service_name, app)