Skip to content

Commit

Permalink
debug workflows otel trace
Browse files Browse the repository at this point in the history 
Signed-off-by: sallyom <[email protected]>
  • Loading branch information
@sallyom
sallyom committed Jun 25, 2024
1 parent e6f4a74 commit d45439b
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 129 deletions.
55 changes: 7 additions & 48 deletions .github/workflows/chatbot.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,10 @@ jobs:
sudo apt-get update
sudo apt-get install -y qemu-user-static
- name: Start build trace
- name: Start job trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="build-image"
export WORKFLOW_NAME="chatbot"
export JOB_NAME="chatbot-build-and-push"
export TRACE_ACTION="start"
python ci/trace-steps.py
Expand All @@ -110,54 +110,19 @@ jobs:
containerfiles: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }}/app/Containerfile
context: recipes/natural_language_processing/${{ env.IMAGE_NAME }}/app

- name: End build trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="build-image"
export TRACE_ACTION="end"
python ci/trace-steps.py
- name: Install Dependencies
working-directory: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }}
run: make install

- name: Start download model trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="download-model"
export TRACE_ACTION="start"
python ci/trace-steps.py
- name: Download model
working-directory: ./models
run: make download-model-granite

- name: End download model trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="download-model"
export TRACE_ACTION="end"
python ci/trace-steps.py
- name: Start functional test run trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="run-functional-tests"
export TRACE_ACTION="start"
python ci/trace-steps.py
- name: Run Functional Tests
shell: bash
run: make functional-tests
working-directory: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }}

- name: End functional test run trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="run-functional-tests"
export TRACE_ACTION="end"
python ci/trace-steps.py
- name: Login to Registry
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: redhat-actions/[email protected]
Expand All @@ -166,13 +131,6 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Start push image trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="push-image"
export TRACE_ACTION="start"
python ci/trace-steps.py
- name: Push Image
id: push_image
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
Expand All @@ -182,9 +140,10 @@ jobs:
tags: ${{ steps.build_image.outputs.tags }}
registry: ${{ env.REGISTRY }}

- name: End push image trace
- name: End job trace
run: |
export WORKFLOW_NAME="chatbot-build-push"
export STEP_NAME="push-image"
export WORKFLOW_NAME="chatbot"
export JOB_NAME="chatbot-build-and-push"
export TRACE_ACTION="end"
python ci/trace-steps.py
28 changes: 7 additions & 21 deletions .github/workflows/test-trace-steps.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
workflow_dispatch:

jobs:
test:
test-build:
if: "!contains(github.event.pull_request.labels.*.name, 'hold-tests')"
runs-on: ubuntu-24.04
steps:
Expand Down Expand Up @@ -66,10 +66,10 @@ jobs:
run: |
./otelcol-contrib --config otel-collector-config.yaml > otel-collector.log 2>&1 &
- name: Start build trace
- name: Start job trace
run: |
export WORKFLOW_NAME="test-workflow"
export STEP_NAME="build"
export WORKFLOW_NAME="test-trace"
export JOB_NAME="test-build"
export TRACE_ACTION="start"
python ci/trace-steps.py
Expand All @@ -78,29 +78,15 @@ jobs:
echo "Simulating build step..."
sleep 2
- name: End build trace
run: |
export WORKFLOW_NAME="test-workflow"
export STEP_NAME="build"
export TRACE_ACTION="end"
python ci/trace-steps.py
- name: Start test trace
run: |
export WORKFLOW_NAME="test-workflow"
export STEP_NAME="test"
export TRACE_ACTION="start"
python ci/trace-steps.py
- name: Test
run: |
echo "Simulating test step..."
sleep 2
- name: End test trace
- name: End job trace
run: |
export WORKFLOW_NAME="test-workflow"
export STEP_NAME="test"
export WORKFLOW_NAME="test-trace"
export JOB_NAME="test-build"
export TRACE_ACTION="end"
python ci/trace-steps.py
Expand Down
97 changes: 37 additions & 60 deletions ci/trace-steps.py
View file
Original file line number Diff line number Diff line change
@@ -1,70 +1,47 @@
import os
import time
from datetime import datetime
from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter

# Initialize Tracer
trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)
span_processor = BatchSpanProcessor(ConsoleSpanExporter())
trace.get_tracer_provider().add_span_processor(span_processor)

def trace_step(step_name: str):
"""
Decorated function that traces execution time.
Args:
step_name (str): Name of the step to trace.
"""
def decorator(func):
def wrapper(*args, **kwargs):
with tracer.start_as_current_span(step_name) as span:
start_time = time.time()
result = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
span.set_attribute("duration_s", duration)
print(f"Step: {step_name}, Duration: {duration}s")
return result
return wrapper
return decorator

def trace_job(job_name: str):
"""
Decorated function that traces execution time.
Args:
job_name (str): Name of the job to trace.
"""
def decorator(func):
def wrapper(*args, **kwargs):
with tracer.start_as_current_span(job_name) as span:
start_time = time.time()
result = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
span.set_attribute("total_duration_s", duration)
print(f"Job: {job_name}, Total Duration: {duration}s")
return result
return wrapper
return decorator
service_name = os.getenv("WORKFLOW_NAME", "default_service")
job_name = os.getenv("JOB_NAME", "default_job")

@trace_step("Example Step")
def example_step():
time.sleep(5) # Simulate work
return "Step Completed: Example Step"

@trace_job("Example Job")
def example_job():
step_result = example_step()
return f"Job Completed: Example Job with {step_result}"
resource = Resource.create({"service.name": service_name})
trace.set_tracer_provider(TracerProvider(resource=resource))
tracer = trace.get_tracer(__name__)
console_span_processor = BatchSpanProcessor(ConsoleSpanExporter())
trace.get_tracer_provider().add_span_processor(console_span_processor)

# Adding OTLP Span Exporter for actual data export
otlp_exporter = OTLPSpanExporter(endpoint="localhost:4317", insecure=True)
otlp_span_processor = BatchSpanProcessor(otlp_exporter)
trace.get_tracer_provider().add_span_processor(otlp_span_processor)

print("Tracer initialized with service name:", service_name)

def set_start_time():
start_time = datetime.now().timestamp()
with open("/tmp/start_time.txt", "w") as file:
file.write(str(start_time))
print("Start time recorded")

def calculate_duration():
with open("/tmp/start_time.txt", "r") as file:
start_time = float(file.read())
end_time = datetime.now().timestamp()
duration = end_time - start_time
print(f"Total Duration: {duration}s")
with tracer.start_as_current_span(job_name) as span:
span.set_attribute("total_duration_s", duration)

if __name__ == "__main__":
job_name = os.getenv("WORKFLOW_NAME", "Default Job")
step_name = os.getenv("STEP_NAME", "Default Step")
trace_action = os.getenv("TRACE_ACTION", "start")

if trace_action == "start":
example_job()
elif trace_action == "end":
print(f"Ending trace for {job_name} - {step_name}")
action = os.getenv("TRACE_ACTION", "start")

if action == "start":
set_start_time()
elif action == "end":
calculate_duration()

0 comments on commit d45439b

Please sign in to comment.