diff --git a/src/splunk_otel/configurator.py b/src/splunk_otel/configurator.py index cf8c7bc..6ee7d62 100644 --- a/src/splunk_otel/configurator.py +++ b/src/splunk_otel/configurator.py @@ -14,12 +14,10 @@ from opentelemetry.sdk._configuration import _OTelSDKConfigurator -from splunk_otel.env import SPLUNK_PROFILER_ENABLED, Env -from splunk_otel.profile import start_profiling +from splunk_otel.profile import _start_profiling_if_enabled class SplunkConfigurator(_OTelSDKConfigurator): def _configure(self, **kwargs): super()._configure(**kwargs) - if Env().is_true(SPLUNK_PROFILER_ENABLED): - start_profiling() + _start_profiling_if_enabled() diff --git a/src/splunk_otel/distro.py b/src/splunk_otel/distro.py index f75f9d7..a15330e 100644 --- a/src/splunk_otel/distro.py +++ b/src/splunk_otel/distro.py @@ -18,16 +18,17 @@ from opentelemetry.instrumentation.propagators import set_global_response_propagator from opentelemetry.sdk.environment_variables import ( OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, OTEL_RESOURCE_ATTRIBUTES, ) from splunk_otel.__about__ import __version__ as version from splunk_otel.env import ( DEFAULTS, - OTEL_LOGS_ENABLED, OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, SPLUNK_ACCESS_TOKEN, SPLUNK_PROFILER_ENABLED, + SPLUNK_PROFILER_LOGS_ENDPOINT, SPLUNK_TRACE_RESPONSE_HEADER_ENABLED, X_SF_TOKEN, Env, @@ -60,8 +61,10 @@ def set_env_defaults(self): def set_profiling_env(self): if self.env.is_true(SPLUNK_PROFILER_ENABLED, "false"): - self.env.setdefault(OTEL_LOGS_ENABLED, "true") self.env.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true") + logs_endpt = self.env.getval(SPLUNK_PROFILER_LOGS_ENDPOINT) + if logs_endpt: + self.env.setval(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, logs_endpt) def set_resource_attributes(self): self.env.list_append(OTEL_RESOURCE_ATTRIBUTES, f"telemetry.distro.name={DISTRO_NAME}") diff --git a/src/splunk_otel/env.py b/src/splunk_otel/env.py index 15bfc10..59658d5 100644 --- a/src/splunk_otel/env.py +++ b/src/splunk_otel/env.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import logging import os DEFAULTS = { @@ -31,18 +31,19 @@ "OTEL_TRACES_SAMPLER": "always_on", } -OTEL_TRACE_ENABLED = "OTEL_TRACE_ENABLED" -OTEL_METRICS_ENABLED = "OTEL_METRICS_ENABLED" -OTEL_LOGS_ENABLED = "OTEL_LOGS_ENABLED" OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED" SPLUNK_OTEL_SYSTEM_METRICS_ENABLED = "SPLUNK_OTEL_SYSTEM_METRICS_ENABLED" SPLUNK_ACCESS_TOKEN = "SPLUNK_ACCESS_TOKEN" # noqa: S105 SPLUNK_TRACE_RESPONSE_HEADER_ENABLED = "SPLUNK_TRACE_RESPONSE_HEADER_ENABLED" SPLUNK_PROFILER_ENABLED = "SPLUNK_PROFILER_ENABLED" +SPLUNK_PROFILER_CALL_STACK_INTERVAL = "SPLUNK_PROFILER_CALL_STACK_INTERVAL" +SPLUNK_PROFILER_LOGS_ENDPOINT = "SPLUNK_PROFILER_LOGS_ENDPOINT" X_SF_TOKEN = "x-sf-token" # noqa S105 +_pylogger = logging.getLogger(__name__) + class Env: """ @@ -65,6 +66,14 @@ def list_append(self, key, value): def getval(self, key, default=""): return self.store.get(key, default) + def getint(self, key, default=0): + val = self.getval(key, str(default)) + try: + return int(val) + except ValueError: + _pylogger.warning("Invalid integer value of '%s' for env var '%s'", val, key) + return default + def setval(self, key, value): self.store[key] = value diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 5430b82..d3b9b5f 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -14,12 +14,15 @@ from opentelemetry.context import Context from opentelemetry.instrumentation.version import __version__ as version from opentelemetry.sdk._logs import LogRecord +from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME from opentelemetry.sdk.resources import Resource from opentelemetry.trace import TraceFlags from opentelemetry.trace.propagation import _SPAN_KEY from splunk_otel import profile_pb2 -from splunk_otel.env import Env +from splunk_otel.env import SPLUNK_PROFILER_CALL_STACK_INTERVAL, SPLUNK_PROFILER_ENABLED, Env + +DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS = 1000 _SERVICE_NAME_ATTR = "service.name" _SPLUNK_DISTRO_VERSION_ATTR = "splunk.distro.version" @@ -32,17 +35,26 @@ _pylogger = logging.getLogger(__name__) -def start_profiling(): +def _start_profiling_if_enabled(env=None): + env = env or Env() + if env.is_true(SPLUNK_PROFILER_ENABLED): + start_profiling(env) + + +def start_profiling(env=None): + env = env or Env() + interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) + svcname = env.getval(OTEL_SERVICE_NAME) + tcm = _ThreadContextMapping() tcm.wrap_context_methods() - period_millis = 100 - resource = _mk_resource(Env().getval("OTEL_SERVICE_NAME")) + resource = _mk_resource(svcname) logger = get_logger("splunk-profiler") - scraper = _ProfileScraper(resource, tcm.get_thread_states(), period_millis, logger) + scraper = _ProfileScraper(resource, tcm.get_thread_states(), interval_millis, logger) global _profile_timer # noqa PLW0603 - _profile_timer = _PeriodicTimer(period_millis, scraper.tick) + _profile_timer = _IntervalTimer(interval_millis, scraper.tick) _profile_timer.start() @@ -142,14 +154,14 @@ def __init__( self, resource, thread_states, - period_millis, + interval_millis, logger: Logger, collect_stacktraces_func=_collect_stacktraces, time_func=time.time, ): self.resource = resource self.thread_states = thread_states - self.period_millis = period_millis + self.interval_millis = interval_millis self.collect_stacktraces = collect_stacktraces_func self.time = time_func self.logger = logger @@ -165,7 +177,7 @@ def mk_log_record(self, stacktraces): time_seconds = self.time() - pb_profile = _stacktraces_to_cpu_profile(stacktraces, self.thread_states, self.period_millis, time_seconds) + pb_profile = _stacktraces_to_cpu_profile(stacktraces, self.thread_states, self.interval_millis, time_seconds) pb_profile_str = _pb_profile_to_str(pb_profile) return LogRecord( @@ -192,9 +204,9 @@ def _pb_profile_to_str(pb_profile) -> str: return b64encoded.decode() -class _PeriodicTimer: - def __init__(self, period_millis, target): - self.period_seconds = period_millis / 1e3 +class _IntervalTimer: + def __init__(self, interval_millis, target): + self.interval_seconds = interval_millis / 1e3 self.target = target self.thread = threading.Thread(target=self._loop, daemon=True) self.sleep = time.sleep @@ -207,7 +219,7 @@ def _loop(self): start_time_seconds = time.time() self.target() elapsed_seconds = time.time() - start_time_seconds - sleep_seconds = max(0, self.period_seconds - elapsed_seconds) + sleep_seconds = max(0, self.interval_seconds - elapsed_seconds) time.sleep(sleep_seconds) def stop(self): @@ -277,7 +289,7 @@ def _extract_stack_summary(frame): return out -def _stacktraces_to_cpu_profile(stacktraces, thread_states, period_millis, time_seconds): +def _stacktraces_to_cpu_profile(stacktraces, thread_states, interval_millis, time_seconds): str_table = _StringTable() locations_table = OrderedDict() functions_table = OrderedDict() @@ -294,7 +306,7 @@ def _stacktraces_to_cpu_profile(stacktraces, thread_states, period_millis, time_ event_period_label = profile_pb2.Label() event_period_label.key = event_period_key - event_period_label.num = period_millis + event_period_label.num = interval_millis samples = [] for stacktrace in stacktraces: diff --git a/tests/ott_profile.py b/tests/ott_profile.py index f91d422..c007d39 100644 --- a/tests/ott_profile.py +++ b/tests/ott_profile.py @@ -1,5 +1,3 @@ -from oteltest import Telemetry -from oteltest.telemetry import count_logs, has_log_attribute from ott_lib import project_path, trace_loop if __name__ == "__main__": @@ -10,10 +8,12 @@ class ProfileOtelTest: def environment_variables(self): return { "SPLUNK_PROFILER_ENABLED": "true", + "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "500", # not necessary, defaults to "1000" (ms) + "SPLUNK_PROFILER_LOGS_ENDPOINT": "http://localhost:4317", # not necessary, this is the default } def requirements(self): - return [project_path(), "oteltest"] + return (project_path(),) def wrapper_command(self): return "opentelemetry-instrument" @@ -21,7 +21,9 @@ def wrapper_command(self): def on_start(self): pass - def on_stop(self, tel: Telemetry, stdout: str, stderr: str, returncode: int): + def on_stop(self, tel, stdout: str, stderr: str, returncode: int): + from oteltest.telemetry import count_logs, has_log_attribute + assert count_logs(tel) assert has_log_attribute(tel, "profiling.data.format") diff --git a/tests/test_distro.py b/tests/test_distro.py index 0a4c8b4..1ea9cbb 100644 --- a/tests/test_distro.py +++ b/tests/test_distro.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from opentelemetry.instrumentation.propagators import get_global_response_propagator, set_global_response_propagator +from opentelemetry.instrumentation.propagators import ( + get_global_response_propagator, + set_global_response_propagator, +) from splunk_otel.__about__ import __version__ as version from splunk_otel.distro import SplunkDistro from splunk_otel.env import Env @@ -72,7 +75,6 @@ def test_server_timing_resp_prop_false(): def test_profiling_enabled(): env_store = {"SPLUNK_PROFILER_ENABLED": "true"} configure_distro(env_store) - assert env_store["OTEL_LOGS_ENABLED"] == "true" assert env_store["OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"] == "true" @@ -90,6 +92,15 @@ def test_profiling_notset(): assert "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED" not in env_store +def test_profiling_endpt(): + env_store = { + "SPLUNK_PROFILER_ENABLED": "true", + "SPLUNK_PROFILER_LOGS_ENDPOINT": "my-logs-endpoint", + } + configure_distro(env_store) + assert "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" in env_store + + def test_resource_attributes(): env_store = {"OTEL_RESOURCE_ATTRIBUTES": "foo=bar"} configure_distro(env_store) diff --git a/tests/test_env.py b/tests/test_env.py index 767a6f5..a14df72 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from splunk_otel.env import Env @@ -19,6 +20,7 @@ def test_env(): e = Env() e.store = { "PREEXISTING": "preexisting", + "FAVORITE_NUMBER": "42", } e.setdefault("PREEXISTING", "default") @@ -40,3 +42,16 @@ def test_env(): assert e.getval("MY_LIST") == "a" e.list_append("MY_LIST", "b") assert e.getval("MY_LIST") == "a,b" + + assert e.getint("FAVORITE_NUMBER", 111) == 42 + assert e.getint("NOT_SET", 222) == 222 + + +def test_get_invalid_int(caplog): + with caplog.at_level(logging.WARNING): + e = Env() + e.store = { + "FAVORITE_NUMBER": "bar", + } + assert e.getint("FAVORITE_NUMBER", 111) == 111 + assert "Invalid integer value" in caplog.text