Add support for specified profiler env vars (#566)

* Add support for env vars in spec:
* SPLUNK_PROFILER_LOGS_ENDPOINT
* SPLUNK_PROFILER_CALL_STACK_INTERVAL

* Lint

* Add and use env getint for profiling interval

* Lint

* Set logs endpoint in oteltest

* Add assertion for getint

src/splunk_otel/configurator.py

@@ -14,12 +14,10 @@
 
 from opentelemetry.sdk._configuration import _OTelSDKConfigurator
 
-from splunk_otel.env import SPLUNK_PROFILER_ENABLED, Env
-from splunk_otel.profile import start_profiling
+from splunk_otel.profile import _start_profiling_if_enabled
 
 
 class SplunkConfigurator(_OTelSDKConfigurator):
     def _configure(self, **kwargs):
         super()._configure(**kwargs)
-        if Env().is_true(SPLUNK_PROFILER_ENABLED):
-            start_profiling()
+        _start_profiling_if_enabled()

src/splunk_otel/distro.py

@@ -18,16 +18,17 @@
 from opentelemetry.instrumentation.propagators import set_global_response_propagator
 from opentelemetry.sdk.environment_variables import (
     OTEL_EXPORTER_OTLP_HEADERS,
+    OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
     OTEL_RESOURCE_ATTRIBUTES,
 )
 
 from splunk_otel.__about__ import __version__ as version
 from splunk_otel.env import (
     DEFAULTS,
-    OTEL_LOGS_ENABLED,
     OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
     SPLUNK_ACCESS_TOKEN,
     SPLUNK_PROFILER_ENABLED,
+    SPLUNK_PROFILER_LOGS_ENDPOINT,
     SPLUNK_TRACE_RESPONSE_HEADER_ENABLED,
     X_SF_TOKEN,
     Env,
@@ -60,8 +61,10 @@ def set_env_defaults(self):
 
     def set_profiling_env(self):
         if self.env.is_true(SPLUNK_PROFILER_ENABLED, "false"):
-            self.env.setdefault(OTEL_LOGS_ENABLED, "true")
             self.env.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
+            logs_endpt = self.env.getval(SPLUNK_PROFILER_LOGS_ENDPOINT)
+            if logs_endpt:
+                self.env.setval(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, logs_endpt)
 
     def set_resource_attributes(self):
         self.env.list_append(OTEL_RESOURCE_ATTRIBUTES, f"telemetry.distro.name={DISTRO_NAME}")

src/splunk_otel/env.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
+import logging
 import os
 
 DEFAULTS = {
@@ -31,18 +31,19 @@
     "OTEL_TRACES_SAMPLER": "always_on",
 }
 
-OTEL_TRACE_ENABLED = "OTEL_TRACE_ENABLED"
-OTEL_METRICS_ENABLED = "OTEL_METRICS_ENABLED"
-OTEL_LOGS_ENABLED = "OTEL_LOGS_ENABLED"
 OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"
 
 SPLUNK_OTEL_SYSTEM_METRICS_ENABLED = "SPLUNK_OTEL_SYSTEM_METRICS_ENABLED"
 SPLUNK_ACCESS_TOKEN = "SPLUNK_ACCESS_TOKEN"  # noqa: S105
 SPLUNK_TRACE_RESPONSE_HEADER_ENABLED = "SPLUNK_TRACE_RESPONSE_HEADER_ENABLED"
 SPLUNK_PROFILER_ENABLED = "SPLUNK_PROFILER_ENABLED"
+SPLUNK_PROFILER_CALL_STACK_INTERVAL = "SPLUNK_PROFILER_CALL_STACK_INTERVAL"
+SPLUNK_PROFILER_LOGS_ENDPOINT = "SPLUNK_PROFILER_LOGS_ENDPOINT"
 
 X_SF_TOKEN = "x-sf-token"  # noqa S105
 
+_pylogger = logging.getLogger(__name__)
+
 
 class Env:
     """
@@ -65,6 +66,14 @@ def list_append(self, key, value):
     def getval(self, key, default=""):
         return self.store.get(key, default)
 
+    def getint(self, key, default=0):
+        val = self.getval(key, str(default))
+        try:
+            return int(val)
+        except ValueError:
+            _pylogger.warning("Invalid integer value of '%s' for env var '%s'", val, key)
+            return default
+
     def setval(self, key, value):
         self.store[key] = value
 

src/splunk_otel/profile.py

@@ -14,12 +14,15 @@
 from opentelemetry.context import Context
 from opentelemetry.instrumentation.version import __version__ as version
 from opentelemetry.sdk._logs import LogRecord
+from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.trace import TraceFlags
 from opentelemetry.trace.propagation import _SPAN_KEY
 
 from splunk_otel import profile_pb2
-from splunk_otel.env import Env
+from splunk_otel.env import SPLUNK_PROFILER_CALL_STACK_INTERVAL, SPLUNK_PROFILER_ENABLED, Env
+
+DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS = 1000
 
 _SERVICE_NAME_ATTR = "service.name"
 _SPLUNK_DISTRO_VERSION_ATTR = "splunk.distro.version"
@@ -32,17 +35,26 @@
 _pylogger = logging.getLogger(__name__)
 
 
-def start_profiling():
+def _start_profiling_if_enabled(env=None):
+    env = env or Env()
+    if env.is_true(SPLUNK_PROFILER_ENABLED):
+        start_profiling(env)
+
+
+def start_profiling(env=None):
+    env = env or Env()
+    interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS)
+    svcname = env.getval(OTEL_SERVICE_NAME)
+
     tcm = _ThreadContextMapping()
     tcm.wrap_context_methods()
 
-    period_millis = 100
-    resource = _mk_resource(Env().getval("OTEL_SERVICE_NAME"))
+    resource = _mk_resource(svcname)
     logger = get_logger("splunk-profiler")
-    scraper = _ProfileScraper(resource, tcm.get_thread_states(), period_millis, logger)
+    scraper = _ProfileScraper(resource, tcm.get_thread_states(), interval_millis, logger)
 
     global _profile_timer  # noqa PLW0603
-    _profile_timer = _PeriodicTimer(period_millis, scraper.tick)
+    _profile_timer = _IntervalTimer(interval_millis, scraper.tick)
     _profile_timer.start()
 
 
@@ -142,14 +154,14 @@ def __init__(
         self,
         resource,
         thread_states,
-        period_millis,
+        interval_millis,
         logger: Logger,
         collect_stacktraces_func=_collect_stacktraces,
         time_func=time.time,
     ):
         self.resource = resource
         self.thread_states = thread_states
-        self.period_millis = period_millis
+        self.interval_millis = interval_millis
         self.collect_stacktraces = collect_stacktraces_func
         self.time = time_func
         self.logger = logger
@@ -165,7 +177,7 @@ def mk_log_record(self, stacktraces):
 
         time_seconds = self.time()
 
-        pb_profile = _stacktraces_to_cpu_profile(stacktraces, self.thread_states, self.period_millis, time_seconds)
+        pb_profile = _stacktraces_to_cpu_profile(stacktraces, self.thread_states, self.interval_millis, time_seconds)
         pb_profile_str = _pb_profile_to_str(pb_profile)
 
         return LogRecord(
@@ -192,9 +204,9 @@ def _pb_profile_to_str(pb_profile) -> str:
     return b64encoded.decode()
 
 
-class _PeriodicTimer:
-    def __init__(self, period_millis, target):
-        self.period_seconds = period_millis / 1e3
+class _IntervalTimer:
+    def __init__(self, interval_millis, target):
+        self.interval_seconds = interval_millis / 1e3
         self.target = target
         self.thread = threading.Thread(target=self._loop, daemon=True)
         self.sleep = time.sleep
@@ -207,7 +219,7 @@ def _loop(self):
             start_time_seconds = time.time()
             self.target()
             elapsed_seconds = time.time() - start_time_seconds
-            sleep_seconds = max(0, self.period_seconds - elapsed_seconds)
+            sleep_seconds = max(0, self.interval_seconds - elapsed_seconds)
             time.sleep(sleep_seconds)
 
     def stop(self):
@@ -277,7 +289,7 @@ def _extract_stack_summary(frame):
     return out
 
 
-def _stacktraces_to_cpu_profile(stacktraces, thread_states, period_millis, time_seconds):
+def _stacktraces_to_cpu_profile(stacktraces, thread_states, interval_millis, time_seconds):
     str_table = _StringTable()
     locations_table = OrderedDict()
     functions_table = OrderedDict()
@@ -294,7 +306,7 @@ def _stacktraces_to_cpu_profile(stacktraces, thread_states, period_millis, time_
 
     event_period_label = profile_pb2.Label()
     event_period_label.key = event_period_key
-    event_period_label.num = period_millis
+    event_period_label.num = interval_millis
 
     samples = []
     for stacktrace in stacktraces:

tests/ott_profile.py

@@ -1,5 +1,3 @@
-from oteltest import Telemetry
-from oteltest.telemetry import count_logs, has_log_attribute
 from ott_lib import project_path, trace_loop
 
 if __name__ == "__main__":
@@ -10,18 +8,22 @@ class ProfileOtelTest:
     def environment_variables(self):
         return {
             "SPLUNK_PROFILER_ENABLED": "true",
+            "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "500",  # not necessary, defaults to "1000" (ms)
+            "SPLUNK_PROFILER_LOGS_ENDPOINT": "http://localhost:4317",  # not necessary, this is the default
         }
 
     def requirements(self):
-        return [project_path(), "oteltest"]
+        return (project_path(),)
 
     def wrapper_command(self):
         return "opentelemetry-instrument"
 
     def on_start(self):
         pass
 
-    def on_stop(self, tel: Telemetry, stdout: str, stderr: str, returncode: int):
+    def on_stop(self, tel, stdout: str, stderr: str, returncode: int):
+        from oteltest.telemetry import count_logs, has_log_attribute
+
         assert count_logs(tel)
         assert has_log_attribute(tel, "profiling.data.format")
 

tests/test_distro.py

@@ -11,7 +11,10 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from opentelemetry.instrumentation.propagators import get_global_response_propagator, set_global_response_propagator
+from opentelemetry.instrumentation.propagators import (
+    get_global_response_propagator,
+    set_global_response_propagator,
+)
 from splunk_otel.__about__ import __version__ as version
 from splunk_otel.distro import SplunkDistro
 from splunk_otel.env import Env
@@ -72,7 +75,6 @@ def test_server_timing_resp_prop_false():
 def test_profiling_enabled():
     env_store = {"SPLUNK_PROFILER_ENABLED": "true"}
     configure_distro(env_store)
-    assert env_store["OTEL_LOGS_ENABLED"] == "true"
     assert env_store["OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"] == "true"
 
 
@@ -90,6 +92,15 @@ def test_profiling_notset():
     assert "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED" not in env_store
 
 
+def test_profiling_endpt():
+    env_store = {
+        "SPLUNK_PROFILER_ENABLED": "true",
+        "SPLUNK_PROFILER_LOGS_ENDPOINT": "my-logs-endpoint",
+    }
+    configure_distro(env_store)
+    assert "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" in env_store
+
+
 def test_resource_attributes():
     env_store = {"OTEL_RESOURCE_ATTRIBUTES": "foo=bar"}
     configure_distro(env_store)

tests/test_env.py

@@ -11,6 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+import logging
 
 from splunk_otel.env import Env
 
@@ -19,6 +20,7 @@ def test_env():
     e = Env()
     e.store = {
         "PREEXISTING": "preexisting",
+        "FAVORITE_NUMBER": "42",
     }
 
     e.setdefault("PREEXISTING", "default")
@@ -40,3 +42,16 @@ def test_env():
     assert e.getval("MY_LIST") == "a"
     e.list_append("MY_LIST", "b")
     assert e.getval("MY_LIST") == "a,b"
+
+    assert e.getint("FAVORITE_NUMBER", 111) == 42
+    assert e.getint("NOT_SET", 222) == 222
+
+
+def test_get_invalid_int(caplog):
+    with caplog.at_level(logging.WARNING):
+        e = Env()
+        e.store = {
+            "FAVORITE_NUMBER": "bar",
+        }
+        assert e.getint("FAVORITE_NUMBER", 111) == 111
+    assert "Invalid integer value" in caplog.text