Add loki

applications/base/grafana/datasources/datasources.yaml

@@ -25,6 +25,12 @@ data:
         url: http://kube-prometheus-stack-prometheus-seedbox.prometheus.svc:9090
         version: 1
 
+      - name: loki
+        type: loki
+        access: proxy
+        url: http://loki-gateway.loki.svc:80
+        version: 1
+
       - name: grafanacloud-traces
         type: tempo
         access: proxy

applications/base/loki/application.libsonnet

@@ -0,0 +1,14 @@
+local app = import '../../lib/app.libsonnet';
+local ns = import 'namespace.libsonnet';
+
+app.new(
+  name='loki',
+  path='applications/base/loki',
+  namespace=ns.metadata.name,
+).withChart(
+  name='loki',
+  repoURL='https://grafana.github.io/helm-charts',
+  targetRevision='5.23.1',
+  releaseName='loki',
+  values='values.yaml'
+)

applications/base/loki/bucket.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: loki-bucket
+spec:
+  bucketName: loki
+  storageClassName: ceph-bucket

applications/base/loki/main.jsonnet

@@ -0,0 +1,7 @@
+// jsonnet base/loki/main.jsonnet -J vendor
+
+local ns = import 'namespace.libsonnet';
+local bucket = std.parseYaml(importstr 'bucket.yaml');
+local rules = import 'rules/rules.libsonnet';
+
+[ns] + bucket + rules

applications/base/loki/namespace.libsonnet

@@ -0,0 +1,3 @@
+local k = import '../../lib/k.libsonnet';
+
+k.core.v1.namespace.new('loki')

applications/base/loki/rules/rules.libsonnet

@@ -0,0 +1,6 @@
+local k = import '../../../lib/k.libsonnet';
+
+[
+  k.core.v1.configMap.new('loki-rules', data={
+  }),
+]

applications/base/loki/values.yaml

@@ -0,0 +1,152 @@
+loki:
+  structuredConfig:
+    auth_enabled: false
+
+    server:
+      log_level: info
+      http_listen_port: 3100
+      grpc_listen_port: 9095
+
+    memberlist:
+      join_members: ["loki-memberlist"]
+
+    limits_config:
+      retention_period: 14d
+      enforce_metric_name: false
+      reject_old_samples: true
+      reject_old_samples_max_age: 168h
+      max_cache_freshness_per_query: 10m
+      split_queries_by_interval: 15m
+      ingestion_rate_mb: 50
+      ingestion_burst_size_mb: 1000
+      per_stream_rate_limit: 5MB
+      per_stream_rate_limit_burst: 20MB
+      shard_streams:
+        enabled: true
+
+    schema_config:
+      configs:
+        - from: "2022-01-11" # quote
+          store: boltdb-shipper
+          object_store: s3
+          schema: v12
+          index:
+            prefix: loki_index_
+            period: 24h
+
+    common:
+      path_prefix: /var/loki
+      replication_factor: 2
+      storage:
+        s3:
+          bucketnames: ${BUCKET_NAME}
+          endpoint: ${BUCKET_HOST}
+          access_key_id: ${AWS_ACCESS_KEY_ID}
+          secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+          insecure: true
+          s3forcepathstyle: true
+      ring:
+        kvstore:
+          store: memberlist
+
+    ruler:
+      enable_api: true
+      enable_alertmanager_v2: true
+      alertmanager_url: http://alertmanager-operated.prometheus.svc.cluster.local:9093
+      storage:
+        type: local
+        local:
+          directory: /rules
+      rule_path: /tmp/scratch
+      ring:
+        kvstore:
+          store: memberlist
+
+    distributor:
+      ring:
+        kvstore:
+          store: memberlist
+
+    compactor:
+      working_directory: /var/loki/boltdb-shipper-compactor
+      shared_store: s3
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 150
+
+    ingester:
+      max_chunk_age: 1h
+      lifecycler:
+        ring:
+          kvstore:
+            store: memberlist
+
+    analytics:
+      reporting_enabled: false
+
+gateway:
+  replicas: 2
+  ingress:
+    enabled: false
+
+read:
+  replicas: 2
+  persistence:
+    storageClass: ~
+    size: 10Gi
+  extraVolumeMounts:
+    - name: rules
+      mountPath: /rules
+  extraVolumes:
+    - name: rules
+      emptyDir: {}
+
+write:
+  replicas: 2
+  persistence:
+    storageClass: ~
+    size: 10Gi
+
+backend:
+  replicas: 2
+  persistence:
+    storageClass: ~
+    size: 10Gi
+  extraArgs:
+    - -config.expand-env=true
+  extraEnvFrom:
+    - configMapRef:
+        name: loki-bucket
+    - secretRef:
+        name: loki-bucket
+  extraVolumeMounts:
+    - name: rules
+      mountPath: /rules/fake
+    - name: scratch
+      mountPath: /tmp/scratch
+  extraVolumes:
+    - name: rules
+      configMap:
+        name: loki-rules
+    - name: scratch
+      emptyDir: {}
+
+monitoring:
+  dashboards:
+    annotations:
+      grafana_dashboard: "1"
+      grafana_folder: Loki
+  serviceMonitor:
+    enabled: true
+    metricsInstance:
+      enabled: false
+  selfMonitoring:
+    enabled: false
+    grafanaAgent:
+      installOperator: false
+  lokiCanary:
+    enabled: true
+
+test:
+  enabled: false

applications/base/promtail/application.libsonnet

@@ -0,0 +1,14 @@
+local app = import '../../lib/app.libsonnet';
+local ns = import 'namespace.libsonnet';
+
+app.new(
+  name='promtail',
+  path='applications/base/promtail',
+  namespace=ns.metadata.name,
+).withChart(
+  name='promtail',
+  repoURL='https://grafana.github.io/helm-charts',
+  targetRevision='6.15.2',
+  releaseName='promtail',
+  values='values.yaml'
+)

applications/base/promtail/main.jsonnet

@@ -0,0 +1,3 @@
+// jsonnet base/promtail/main.jsonnet -J vendor
+
+[]

applications/base/promtail/namespace.libsonnet

@@ -0,0 +1 @@
+import '../loki/namespace.libsonnet'

applications/base/promtail/values.yaml

@@ -0,0 +1,29 @@
+serviceMonitor:
+  enabled: true
+  prometheusRule:
+    enabled: true
+    rules:
+      - alert: PromtailRequestErrors
+        expr: 100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance) > 10
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          description: |
+            The {{ $labels.job }} {{ $labels.route }} is experiencing
+            {{ printf "%.2f" $value }} errors.
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
+          summary: Promtail request errors (instance {{ $labels.instance }})
+      - alert: PromtailRequestLatency
+        expr: histogram_quantile(0.99, sum(rate(promtail_request_duration_seconds_bucket[5m])) by (le)) > 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Promtail request latency (instance {{ $labels.instance }})
+          description: |
+            The {{ $labels.job }} {{ $labels.route }} is experiencing
+            {{ printf "%.2f" $value }}s 99th percentile latency.
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}

applications/environments/home/imports.libsonnet

@@ -17,6 +17,7 @@
   import 'k8s-event-logger/application.libsonnet',
   import 'k8up/application.libsonnet',
   import 'local-ai/application.libsonnet',
+  import 'loki/application.libsonnet',
   import 'metallb/application.libsonnet',
   import 'metrics-server/application.libsonnet',
   import 'openspeedtest/application.libsonnet',
@@ -25,6 +26,7 @@
   import 'pgadmin/application.libsonnet',
   import 'prometheus/application.libsonnet',
   import 'prometheus-stack/application.libsonnet',
+  import 'promtail/application.libsonnet',
   import 'rclone-restic/application.libsonnet',
   import 'robusta/application.libsonnet',
   import 'rook-ceph-cluster/application.libsonnet',

applications/environments/home/loki/application.libsonnet

@@ -0,0 +1 @@
+import '../../../base/loki/application.libsonnet'

applications/environments/home/promtail/application.libsonnet

@@ -0,0 +1 @@
+import '../../../base/promtail/application.libsonnet'