Merge remote-tracking branch 'origin/main' into davidgrant/bb-obs

.github/workflows/sbom-report.yml

@@ -14,7 +14,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Anchore SBOM Action
-      uses: anchore/[email protected].2
+      uses: anchore/[email protected].3
       with:
          artifact-name: ${{ github.event.repository.name }}-spdx.json
 

ADOPTERS.md

@@ -5,3 +5,4 @@ If you’re using Grafana Mimir, feel free to add yourself or your organization.
 - [Chaordic](https://chaordic.io)
 - [SysEleven GmbH](https://www.syseleven.de)
 - [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/)
+- [Giant Swarm](https://www.giantswarm.io/)

CHANGELOG.md

@@ -13,7 +13,7 @@
   * `cortex_alertmanager_state_replication_failed_total`
   * `cortex_alertmanager_alerts`
   * `cortex_alertmanager_silences`
-* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #9367 #9368 #9398 #9399 #9403 #9417 #9418 #9419 #9420 #9482 #9504 #9505 #9507 #9518 #9531 #9532 #9533 #9553
+* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #9367 #9368 #9398 #9399 #9403 #9417 #9418 #9419 #9420 #9482 #9504 #9505 #9507 #9518 #9531 #9532 #9533 #9553 #9558 #9588 #9589
 * [FEATURE] Query-frontend: added experimental configuration options `query-frontend.cache-errors` and `query-frontend.results-cache-ttl-for-errors` to allow non-transient responses to be cached. When set to `true` error responses from hitting limits or bad data are cached for a short TTL. #9028
 * [FEATURE] gRPC: Support S2 compression. #9322
   * `-alertmanager.alertmanager-client.grpc-compression=s2`
@@ -49,6 +49,7 @@
 
 ### Mixin
 
+* [ENHANCEMENT] Unify ingester autoscaling panels on 'Mimir / Writes' dashboard to work for both ingest-storage and non-ingest-storage autoscaling. #9617
 * [BUGFIX] Dashboards: Fix autoscaling metrics joins when series churn. #9412 #9450 #9432
 * [BUGFIX] Alerts: Fix autoscaling metrics joins in `MimirAutoscalerNotActive` when series churn. #9412
 

RELEASE.md

@@ -104,14 +104,15 @@ If something is not clear, you can get back to this document to learn more about
     ```bash
     ./tools/release/create-pr-to-merge-release-branch-to-main.sh
     ```
+    This prepares a PR into `main` branch. On approval, **use** the `merge-approved-pr-branch-to-main.sh` script, following the [instruction](https://github.com/grafana/mimir/blob/main/RELEASE.md#merging-release-branch-into-main) on how to merge the PR with "Merge commit" (i.e. we DO NOT "Squash and merge" this one).
   - [ ] Publish the Github pre-release draft after getting review from at least one maintainer
   - [ ] Announce the release candidate on social media such as on Mimir community slack using your own Twitter, Mastodon or LinkedIn account
 - [ ] Vendor the release commit of Mimir into Grafana Enterprise Metrics (GEM)
   - _This is addressed by Grafana Labs_
 - [ ] Publish a `mimir-distributed` Helm chart release candidate. Follow the instructions in [Release process for a release candidate](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/RELEASE.md#release-process-for-a-release-candidate)
-- [ ] Promote experimental features to experimental and remove deprecated features for the **next** release:
-  - [ ] Open a PR for every experimental feature we want to promote to stable
-  - [ ] Open a PR to remove any deprecated feature or configuration option that should be removed in the next release
+- [ ] Promote experimental features to stable and remove deprecated features for the **next** release:
+  - [ ] Open a PR into `main` branch for every experimental feature we want to promote to stable
+  - [ ] Open a PR into `main` branch with any deprecated feature or configuration option removed in the next release
 
 ### Publish the stable release
 
@@ -138,6 +139,7 @@ If something is not clear, you can get back to this document to learn more about
     ```bash
     ./tools/release/create-pr-to-merge-release-branch-to-main.sh
     ```
+    This prepares a PR into `main` branch. On approval, **use** the `merge-approved-pr-branch-to-main.sh` script, following the [instruction](https://github.com/grafana/mimir/blob/main/RELEASE.md#merging-release-branch-into-main) on how to merge the PR with "Merge commit" (i.e. we DO NOT "Squash and merge" this one).
   - [ ] If during the release process settings in the `renovate.json5` have been modified in such a way that dependency updates maintain more than the latest two minor versions,
         modify it again to ensure that only the latest two minor versions get updated.
         For instance, if versions 3.1, 3.0 and 2.10 are configured in `renovate.json5`, `renovate.json5` should keep updated the following branches:
@@ -148,6 +150,7 @@ If something is not clear, you can get back to this document to learn more about
   - [ ] Open a PR to update the mixin in ["Self-hosted Grafana Mimir" integration](https://grafana.com/docs/grafana-cloud/monitor-infrastructure/integrations/integration-reference/integration-mimir/)
     - _This is addressed by Grafana Labs_
   - [ ] [Publish dashboards to grafana.com](https://github.com/grafana/mimir/blob/main/RELEASE.md#publish-a-stable-release)
+    - _This is addressed by Grafana Labs_
   - [ ] After publishing a GEM release publish the `mimir-distributed` Helm chart. Follow the instructions in [Release process for a final release](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/RELEASE.md#release-process-for-a-final-release)
 ````
 

docs/sources/helm-charts/mimir-distributed/_index.md

@@ -8,8 +8,8 @@ keywords:
   - Grafana Enterprise Metrics
   - Grafana metrics
 cascade:
-  MIMIR_VERSION: "v2.13.x"
-  GEM_VERSION: "v2.13.x"
+  MIMIR_VERSION: "v2.14.x"
+  GEM_VERSION: "v2.14.x"
   ALLOY_VERSION: "latest"
 ---
 

docs/sources/mimir/get-started/play-with-grafana-mimir/index.md

@@ -24,8 +24,22 @@ keywords:
 menuTitle: Play with Mimir
 title: Play with Mimir
 weight: 10
+killercoda:
+  title: Play with Mimir
+  description: Learn about Grafana Mimir, which provides distributed, horizontally scalable, and highly available long term storage for Prometheus.
+  details:
+    intro:
+      foreground: "docker-compose-update.sh"
+  preprocessing:
+    substitutions:
+      - regexp: "tutorial-architecture\\.png"
+        replacement: https://grafana.com/media/docs/mimir/tutorial-architecture.png
+  backend:
+    imageid: ubuntu
 ---
 
+<!-- INTERACTIVE page intro.md START -->
+
 # Play with Mimir
 
 Grafana Mimir is a distributed, horizontally scalable, and highly available long term storage for [Prometheus](https://prometheus.io).
@@ -37,12 +51,27 @@ In this tutorial, you'll:
 - Run Grafana to explore Grafana Mimir dashboards
 - Configure a testing recording rule and alert in Grafana Mimir
 
+<!-- INTERACTIVE ignore START -->
+
 ## Prerequisites
 
 - Git
 - [Docker](https://docs.docker.com/) and [Docker Compose](https://docs.docker.com/compose/)
 - Availability of both ports `9000` and `9009` on your host machine
 
+{{< admonition type="tip" >}}
+Alternatively, you can try out this example in our interactive learning environment: [Play with Mimir](https://killercoda.com/grafana-labs/course/mimir/play-with-mimir).
+
+It's a fully configured environment with all dependencies already installed.
+
+{{< /admonition >}}
+
+<!-- INTERACTIVE ignore END -->
+
+<!-- INTERACTIVE page intro.md END -->
+
+<!-- INTERACTIVE page step1.md START -->
+
 ## Download tutorial configuration
 
 1. Create a copy of the Grafana Mimir repository using the Git command line:
@@ -55,14 +84,20 @@ In this tutorial, you'll:
    cd docs/sources/mimir/get-started/play-with-grafana-mimir/
    ```
 
-**Note**: the instructions in this tutorial assume that your working directory is `docs/sources/mimir/get-started/play-with-grafana-mimir/`.
+{{< admonition type="note" >}}
+The instructions in this tutorial assume that your working directory is `docs/sources/mimir/get-started/play-with-grafana-mimir/`.
+{{< /admonition >}}
+
+<!-- INTERACTIVE page step1.md END -->
+
+<!-- INTERACTIVE page step2.md START -->
 
 ## Start Grafana Mimir and dependencies
 
 Start running your local setup with the following Docker command:
 
 ```bash
-docker-compose up -d
+docker compose up -d
 ```
 
 This command starts:
@@ -90,8 +125,18 @@ The following ports will be exposed on the host:
 
 To learn more about the Grafana Mimir configuration, you can review the configuration file `config/mimir.yaml`.
 
+<!-- INTERACTIVE page step2.md END -->
+
+<!-- INTERACTIVE page step3.md START -->
+
 ## Explore Grafana Mimir dashboards
 
+{{< docs/ignore >}}
+{{< admonition type="note" >}}
+Sandbox users: If you're using the interactive learning environment, you can access all links directly by clicking on them. This will redirect you to the VM's localhost where the services are running.
+{{< /admonition >}}
+{{< /docs/ignore >}}
+
 Open Grafana on your local host [`http://localhost:9000`](http://localhost:9000) and view dashboards showing the status
 and health of your Grafana Mimir cluster. The dashboards query Grafana Mimir for the metrics they display.
 
@@ -109,6 +154,10 @@ A couple of caveats:
 
 The dashboards installed in the Grafana are taken from the Grafana Mimir mixin which packages up Grafana Labs' best practice dashboards, recording rules, and alerts for monitoring Grafana Mimir. To learn more about the mixin, check out the Grafana Mimir mixin documentation. To learn more about how Grafana is connecting to Grafana Mimir, review the [Mimir datasource](http://localhost:9000/datasources).
 
+<!-- INTERACTIVE page step3.md END -->
+
+<!-- INTERACTIVE page step4.md START -->
+
 ## Configure your first recording rule
 
 Recording rules allow you to precompute frequently needed or computationally expensive expressions and save their result
@@ -118,9 +167,8 @@ offered by Grafana.
 1. Open [Grafana Alerting](http://localhost:9000/alerting/list).
 2. Click **New recording rule**, which also allows you to configure recording rules.
 3. Configure the recording rule:
-   1. Select **Mimir or Loki recording rule** in the top selector.
+   1. Give the rule a name, such as `sum:up`.
    1. Choose **Mimir** in the **Select data source** field.
-   1. Type `sum:up` in the **Rule name** field.
    1. Choose **Code** in the **Builder | Code** field on the right.
    1. Type `sum(up)` in the **Metrics browser** query field.
    1. Type `example-namespace` in the **Namespace** field.
@@ -137,22 +185,28 @@ rule is now being created in Grafana Mimir ruler and will be soon available for
    ```
 1. Confirm the query returns a value of `3` which is the number of Mimir instances currently running in your local setup.
 
+<!-- INTERACTIVE page step4.md END -->
+
+<!-- INTERACTIVE page step5.md START -->
+
 ## Configure your first alert rule
 
 Alerting rules allow you to define alert conditions based on PromQL expressions and to send notifications about firing
 alerts to Grafana Mimir Alertmanager. In this section you're going to configure an alerting rule in Grafana Mimir using
 tooling offered by Grafana.
 
 1. Open [Grafana Alerting](http://localhost:9000/alerting/list).
-2. Click **New alert rule**.
-3. Configure the alert rule:
-   1. Select **Mimir or Loki alert** in the top selector.
-   2. Choose **Mimir** in the **Select data source** field.
-   3. Type `up == 0` in the **Metrics browser** query field.
-   4. Type `MimirNotRunning` in the **Rule name** field.
-   5. Select `example-namespace` in the **Namespace** field.
-   6. Select `example-group` in the **Group** field.
-   7. From the upper-right corner, click the **Save and exit** button.
+1. Click **New alert rule**.
+1. Configure the alert rule:
+   1. Type `MimirNotRunning` in the **Rule name** field.
+   1. Choose **Mimir** in the **Select data source** field.
+   1. Type `count(up == 0)` in the **Metrics browser** query field. This currently shows `no data` since all instances are running.
+1. Scroll down to **Set evaluation behavior**:
+   1. Select `New folder` and type `example-folder` in the **Folder name** field.
+   1. Select `New evaluation group` and type `example-group` in the **Group name** field. Set the evaluation interval to `30s`.
+1. Scroll down to **Configure labels and notifications**:
+   1. Select the `Contract point` dropdown and choose `grafana-default-email`.
+1. Click the **Save rule and exit** button.
 
 Your `MimirNotRunning` alert rule is now being created in Grafana Mimir ruler and is expected to fire when the number of
 Grafana Mimir instances is less than three. You can check its status by opening the [Grafana Alerting](http://localhost:9000/alerting/list)
@@ -162,9 +216,9 @@ To see the alert firing we can introduce an outage in the Grafana Mimir cluster:
 
 1. Abruptly terminate one of the three Grafana Mimir instances:
    ```bash
-   docker-compose kill mimir-3
+   docker compose kill mimir-3
    ```
-1. Open [Grafana Alerting](http://localhost:9000/alerting/list) and check out the state of the alert `MimirNotRunning`,
+2. Open [Grafana Alerting](http://localhost:9000/alerting/list) and check out the state of the alert `MimirNotRunning`,
    which should switch to "Pending" state in about one minute and to "Firing" state after another minute. _Note: since we abruptly
    terminated a Mimir instance, Grafana Alerting UI may temporarily show an error when querying rules: the error will
    auto resolve shortly, as soon as Grafana Mimir internal health checking detects the terminated instance as unhealthy._
@@ -186,16 +240,26 @@ To resolve the alert and recover from the outage, restart the Grafana Mimir inst
    docker-compose start mimir-3
    ```
 2. Open [Grafana Alerting](http://localhost:9000/alerting/list) and check out the state of the alert `MimirNotRunning`,
-   which should switch to "Normal" state in about one minute.
+   which should switch to "Normal" state in about 30 seconds.
+
+<!-- INTERACTIVE page step5.md END -->
+
+<!-- INTERACTIVE page finish.md START -->
 
 ## Summary
 
 In this tutorial you started Grafana Mimir locally in a high-available setup as well as a Prometheus instance that remote wrote
 some metrics to Grafana Mimir. You then queried those metrics stored in Mimir using Grafana, and visualized them in some Grafana dashboards.
 Lastly, you configured a recording rule and an alert via the Grafana Alerting UI and verified that the alert fired as expected when the condition was met.
 
+<!-- INTERACTIVE ignore START -->
+
 Once you've completed the tutorial, release all Docker resources by running this Docker command:
 
 ```bash
 docker-compose down -v
 ```
+
+<!-- INTERACTIVE ignore END -->
+
+<!-- INTERACTIVE page finish.md END -->

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.1
-	github.com/grafana/dskit v0.0.0-20241011202249-8e7752e59bab
+	github.com/grafana/dskit v0.0.0-20241013223235-619c42124e93
 	github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/json-iterator/go v1.1.12
@@ -74,10 +74,10 @@ require (
 	github.com/okzk/sdnotify v0.0.0-20240725214427-1c1fdd37c5ac
 	github.com/prometheus/procfs v0.15.1
 	github.com/shirou/gopsutil/v4 v4.24.8
-	github.com/thanos-io/objstore v0.0.0-20240913165201-fd105025a2e5
+	github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90
 	github.com/twmb/franz-go v1.17.1
 	github.com/twmb/franz-go/pkg/kadm v1.13.0
-	github.com/twmb/franz-go/pkg/kfake v0.0.0-20240821035758-b77dd13e2bfa
+	github.com/twmb/franz-go/pkg/kfake v0.0.0-20241010233600-b66ceb79e585
 	github.com/twmb/franz-go/pkg/kmsg v1.8.0
 	github.com/twmb/franz-go/plugin/kotel v1.5.0
 	github.com/twmb/franz-go/plugin/kprom v1.1.0
@@ -86,7 +86,7 @@ require (
 	go.opentelemetry.io/otel v1.30.0
 	go.opentelemetry.io/otel/trace v1.30.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6
+	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
 	golang.org/x/term v0.25.0
 	google.golang.org/api v0.196.0
 	google.golang.org/protobuf v1.34.2
@@ -282,7 +282,7 @@ require (
 )
 
 // Using a fork of Prometheus with Mimir-specific changes.
-replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241003114040-35ec40c3b27a
+replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241015102654-eaa7eae2d877
 
 // Replace memberlist with our fork which includes some fixes that haven't been
 // merged upstream yet:
@@ -315,7 +315,7 @@ replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-aler
 // - https://github.com/grafana/franz-go/pull/1
 // - https://github.com/grafana/franz-go/pull/3
 // - https://github.com/grafana/franz-go/pull/4
-replace github.com/twmb/franz-go => github.com/grafana/franz-go v0.0.0-20241009101240-fa97d35e871f
+replace github.com/twmb/franz-go => github.com/grafana/franz-go v0.0.0-20241009100846-782ba1442937
 
 // Pin Google GRPC to v1.65.0 as v1.66.0 has API changes and also potentially performance regressions.
 // Following https://github.com/grafana/dskit/pull/581