Local monitoring setup with OTEL and Grafana LGTM-stack

.gitignore

@@ -15,3 +15,9 @@ AltinnPlatformLocal/
 # Visual Studio
 .vs/
 *.user
+
+.mono/
+
+node_modules/
+k6/*
+!k6/*.sample.js

Makefile

@@ -27,4 +27,9 @@ podman-selinux-bind-hack:
 	@echo "Running best effort commands to make bind mounts work on Apple Silicon and Linux with podman. Dirty hack until actual issue is located and fixed."
 	podman container run -v ./testdata/:/testdata/:Z --rm -it --entrypoint cat nginx:alpine-perl /testdata/authorization/claims/1337.json > /dev/null
 	podman container run -v ./loadbalancer/templates/:/testdata/:Z --rm -it --entrypoint cat nginx:alpine-perl /testdata/nginx.conf.conf > /dev/null
-	podman container run -v ./loadbalancer/www/:/testdata/:Z --rm -it --entrypoint cat nginx:alpine-perl /testdata/502App.html > /dev/null
+	podman container run -v ./loadbalancer/www/:/testdata/:Z --rm -it --entrypoint cat nginx:alpine-perl /testdata/502App.html > /dev/null
+
+.PHONY: sync-dashboards
+sync-dashboards:
+	curl -o ./infra/grafana-dashboards/aspnetcore.json https://raw.githubusercontent.com/dotnet/aspire/main/src/Grafana/dashboards/aspnetcore.json
+	curl -o ./infra/grafana-dashboards/aspnetcore-endpoint.json https://raw.githubusercontent.com/dotnet/aspire/main/src/Grafana/dashboards/aspnetcore-endpoint.json

README.md

@@ -7,11 +7,14 @@ These are some of the required steps, tips, and tricks when it comes to running
   - [Clone the repository](#clone-the-repository)
   - [Option A: Start the containers using podman](#option-a-start-the-containers-using-podman)
   - [Option B: Start the containers using Docker](#option-b-start-the-containers-using-docker)
+  - [Option C (preview): Automatic detection](#option-c-preview-automatic-detection)
   - [Start your app](#start-your-app)
 - [Changing configuration](#changing-configuration)
 - [Multiple apps at the same time (running LocalTest locally)](#multiple-apps-at-the-same-time-running-localtest-locally)
 - [Changing test data](#changing-test-data)
+  - [Add a missing role for a test user](#add-a-missing-role-for-a-test-user)
 - [Known issues](#known-issues)
+  - [Localtest reports that the app is not running even though it is](#localtest-reports-that-the-app-is-not-running-even-though-it-is)
 
 ### Prerequisites
 
@@ -54,6 +57,14 @@ Start the containers with the following command:
 podman compose --file podman-compose.yml up -d --build
 ```
 
+Optionally, if you want access to Grafana and a local monitoring setup based on OpenTelemetry:
+
+```shell
+podman compose --file podman-compose.yml --profile "monitoring" up -d --build
+# Grafana should be available at http://local.altinn.cloud:8000/grafana
+# Remember to enable the 'UseOpenTelemetry' configuration flag in the appsettings.json of the app
+```
+
 > [!NOTE]
 > If you are using linux or mac you can use the Makefile to build and run the containers.
 > 
@@ -83,6 +94,14 @@ This mode supports running one app at a time. If you need to run multiple apps a
 ```shell
 docker compose up -d --build
 ```
+
+Optionally, if you want access to Grafana and a local monitoring setup based on OpenTelemetry:
+
+```shell
+docker compose --profile "monitoring" up -d --build
+# Grafana should be available at http://local.altinn.cloud/grafana
+# Remember to enable the 'UseOpenTelemetry' configuration flag in the appsettings.json of the app
+```
 
 > [!NOTE]
 > If you are using linux or mac you can use the Makefile to build and run the containers.
@@ -93,6 +112,28 @@ docker compose up -d --build
 
 Localtest should now be runningn on port 80 and can be accessed on <http://local.altinn.cloud:80>.
 
+#### Option C (preview): Automatic detection
+
+There is a preview helper script that will execute the correct commands in a cross-platform way.
+Either docker or podman must be installed.
+
+```shell
+./run.cmd
+```
+
+Optionally, if you want access to Grafana and a local monitoring setup based on OpenTelemetry:
+
+```shell
+./run.cmd -m
+```
+
+If the localtest setup is already running, it will restart.
+
+To stop localtest
+```shell
+./run.cmd stop
+```
+
 #### Start your app
 _This step requires that you have already [created an app](https://docs.altinn.studio/app/getting-started/create-app/), added a [data model](https://docs.altinn.studio/app/development/data/data-model/data-models-tool/), and [cloned the app](https://docs.altinn.studio/app/getting-started/local-dev/) to your local environment._
 
@@ -216,6 +257,23 @@ This would be required if your app requires a role which none of the test users
 4. Save and close the file
 5. Restart LocalTest
 
+### k6 testing
+
+In the k6 folder there is a sample loadtest that can be adapted to run automated tests against a local app
+It was created to simulate workloads and test monitoring and instrumentation.
+
+```shell
+cp k6/loadtest.sample.js k6/loadtest.js
+# Now make edits to k6/loadtest.js to fit your application
+
+# To run, either
+./run.cmd k6
+# or 
+docker run --rm -i --net=host grafana/k6:master-with-browser run - <k6/loadtest.js
+```
+
+For a decent editing experience, run `npm install` and use a editor with JS support.
+
 ### Known issues
 
 #### Localtest reports that the app is not running even though it is

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.6"
-
 networks:
   altinntestlocal_network:
     external: false
@@ -77,5 +75,92 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  monitoring_otel_collector:
+    container_name: monitoring_otel_collector
+    image: otel/opentelemetry-collector-contrib:0.98.0
+    restart: always
+    networks:
+      - altinntestlocal_network
+    command:
+      - --config=/etc/otel-collector.yaml
+    volumes:
+      - ./infra/otel-collector.yaml:/etc/otel-collector.yaml
+    ports:
+      - "4317:4317"   # OTLP gRPC receiver
+    profiles: ["monitoring"]
+    depends_on:
+      - monitoring_mimir
+      - monitoring_tempo
+      - monitoring_loki
+
+  monitoring_tempo:
+    container_name: monitoring_tempo
+    image: grafana/tempo:2.4.1
+    restart: always
+    networks:
+      - altinntestlocal_network
+    command:
+      - -config.file=/etc/tempo.yaml
+      - -log.level=error
+    volumes:
+      - ./infra/tempo.yaml:/etc/tempo.yaml
+    profiles: ["monitoring"]
+
+  monitoring_mimir:
+    container_name: monitoring_mimir
+    image: grafana/mimir:2.12.0
+    restart: always
+    networks:
+      - altinntestlocal_network
+    command:
+      - -config.file=/etc/mimir.yaml
+      - -target=all
+      - -log.level=error
+    volumes:
+      - ./infra/mimir.yaml:/etc/mimir.yaml
+    profiles: ["monitoring"]
+
+  monitoring_loki:
+    container_name: monitoring_loki
+    image: grafana/loki:3.0.0
+    restart: always
+    networks:
+      - altinntestlocal_network
+    command:
+      - -config.file=/etc/loki.yaml
+      - -target=all
+      - -log.level=error
+    volumes:
+      - ./infra/loki.yaml:/etc/loki.yaml
+    profiles: ["monitoring"]
+
+  monitoring_grafana:
+    container_name: monitoring_grafana
+    image: grafana/grafana:10.4.2
+    restart: always
+    networks:
+      - altinntestlocal_network
+    volumes:
+      - ./infra/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./infra/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml
+      - ./infra/grafana-dashboards:/var/lib/grafana/dashboards
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE="traceqlEditor traceToMetrics metricsSummary"
+      - GF_LOG_LEVEL=error
+      - GF_SERVER_DOMAIN=${TEST_DOMAIN:-local.altinn.cloud}
+      - GF_SERVER_SERVE_FROM_SUB_PATH=true
+      - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s:%(http_port)s/grafana/
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    profiles: ["monitoring"]
+    depends_on:
+      - monitoring_otel_collector
+      - monitoring_mimir
+      - monitoring_tempo
+      - monitoring_loki
+
 volumes:
   AltinnPlatformLocal:

infra/grafana-dashboards.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: "Dashboard provider"
+    orgId: 1
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true