Update terraform and workflow to deploy Reporting V2

.github/actions/configure-metrics-aws/action.yml

@@ -33,7 +33,7 @@ runs:
   - name: Install cert-manager
     shell: bash
     run: |
-      kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml
+      kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml
       kubectl rollout status deployment cert-manager-webhook --namespace=cert-manager --timeout=5m
 
   - name: Install OpenTelemetry operator

.github/actions/configure-metrics/action.yml

@@ -26,7 +26,7 @@ runs:
   - name: Install cert-manager
     shell: bash
     run: |
-      kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml
+      kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml
       kubectl rollout status deployment cert-manager-webhook --namespace=cert-manager --timeout=5m
 
   - name: Install OpenTelemetry operator

.github/workflows/configure-reporting-v2.yml

@@ -0,0 +1,164 @@
+# Copyright 2023 The Cross-Media Measurement Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Configure Reporting V2"
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+        required: true
+      image-tag:
+        description: "Tag of container images"
+        type: string
+        required: true
+      apply:
+        description: "Apply the new configuration"
+        type: boolean
+        required: true
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+        - dev
+      image-tag:
+        description: "Tag of container images"
+        type: string
+        required: true
+      apply:
+        description: "Apply the new configuration"
+        type: boolean
+        default: false
+
+permissions:
+  id-token: write
+
+env:
+  KUSTOMIZATION_PATH: "k8s/reporting_v2"
+
+jobs:
+  update-reporting-v2:
+    runs-on: ubuntu-20.04
+    environment: ${{ inputs.environment }}
+    steps:
+    - uses: actions/checkout@v3
+
+    # Authenticate to Google Cloud. This will export some environment
+    # variables, including GCLOUD_PROJECT.
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v1
+      with:
+        workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }}
+        service_account: ${{ vars.GKE_CONFIG_SERVICE_ACCOUNT }}
+
+    - name: Export BAZEL_BIN
+      run: echo "BAZEL_BIN=$(bazelisk info bazel-bin)" >> $GITHUB_ENV
+
+    - name: Get GKE cluster credentials
+      uses: google-github-actions/get-gke-credentials@v1
+      with:
+        cluster_name: reporting-v2
+        location: ${{ vars.GCLOUD_ZONE }}
+
+    - name: Configure metrics
+      uses: ./.github/actions/configure-metrics
+      if: ${{ inputs.apply }}
+
+    - name: Generate archives
+      env:
+        IMAGE_TAG: ${{ inputs.image-tag }}
+        POSTGRES_INSTANCE: ${{ vars.POSTGRES_INSTANCE }}
+        GCLOUD_REGION: ${{ vars.GCLOUD_REGION }}
+        KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }}
+      run: >
+        bazelisk build 
+        //src/main/k8s/dev:reporting_v2.tar
+        //src/main/k8s/testing/secretfiles:archive
+        --config ghcr
+        --define "image_tag=$IMAGE_TAG"
+        --define "google_cloud_project=$GCLOUD_PROJECT"
+        --define "postgres_instance=$POSTGRES_INSTANCE"
+        --define "postgres_region=$GCLOUD_REGION"
+        --define "kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET"
+
+    - name: Make Kustomization dir
+      run: mkdir -p "$KUSTOMIZATION_PATH"
+
+    - name: Extract Kustomization archive
+      run: >
+        tar -xf "$BAZEL_BIN/src/main/k8s/dev/reporting_v2.tar"
+        -C "$KUSTOMIZATION_PATH"
+
+    - name: Extract secret files archive
+      run: >
+        tar -xf "$BAZEL_BIN/src/main/k8s/testing/secretfiles/archive.tar"
+        -C "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_secrets"
+
+    # Write files from configuration variables. Since it appears that GitHub
+    # configuration variables use DOS (CRLF) line endings, we convert these to
+    # Unix (LF) line endings.
+
+    - name: Write AKID to principal map
+      env:
+        AKID_TO_PRINCIPAL_MAP: ${{ vars.AKID_TO_PRINCIPAL_MAP }}
+      run: >
+        echo "$AKID_TO_PRINCIPAL_MAP" | sed $'s/\r$//'  >
+        "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_config_files/authority_key_identifier_to_principal_map.textproto"
+
+    - name: Write encryption key-pair config
+      env:
+        ENCRYPTION_KEY_PAIR_CONFIG: ${{ vars.ENCRYPTION_KEY_PAIR_CONFIG }}
+      run: >
+        echo "$ENCRYPTION_KEY_PAIR_CONFIG" | sed $'s/\r$//'  >
+        "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_config_files/encryption_key_pair_config.textproto"
+
+    - name: Copy metric spec config
+      run: >
+        cp src/main/k8s/testing/secretfiles/metric_spec_config.textproto
+        "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_config_files/"
+
+    - name: Write measurement consumer config
+      env:
+        MEASUREMENT_CONSUMER_CONFIG: ${{ vars.MEASUREMENT_CONSUMER_CONFIG }}
+      run: >
+        echo "$MEASUREMENT_CONSUMER_CONFIG" | sed $'s/\r$//'  >
+        "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_secrets/measurement_consumer_config.textproto"
+
+    - name: Copy secret generator
+      run: >
+        cp src/main/k8s/testing/secretfiles/reporting_secrets_kustomization.yaml
+        "$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2_secrets/kustomization.yaml"
+
+    - name: Export KUSTOMIZE_PATH
+      run: echo "KUSTOMIZE_PATH=$KUSTOMIZATION_PATH/src/main/k8s/dev/reporting_v2" >> $GITHUB_ENV
+
+    # Run kubectl diff, treating the command as succeeded even if the exit
+    # code is 1 as kubectl uses this code to indicate there's a diff.
+    - name: kubectl diff
+      id: kubectl-diff
+      run: kubectl diff -k "$KUSTOMIZE_PATH" || (( $? == 1 ))
+
+    - name: kubectl apply
+      if: ${{ inputs.apply }}
+      run: kubectl apply -k "$KUSTOMIZE_PATH"
+
+    - name: Wait for rollout
+      if: ${{ inputs.apply }}
+      run: |
+        for deployment in $(kubectl get deployments -o name); do
+          kubectl rollout status "$deployment" --timeout=5m
+        done

.github/workflows/configure-reporting.yml

@@ -86,7 +86,7 @@ jobs:
         KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }}
       run: >
         bazelisk build 
-        "//src/main/k8s/dev:reporting.tar"
+        //src/main/k8s/dev:reporting.tar
         //src/main/k8s/testing/secretfiles:archive
         --config ghcr
         --define "image_tag=$IMAGE_TAG"

.github/workflows/update-cmms.yml

@@ -101,6 +101,14 @@ jobs:
       environment: ${{ inputs.environment }}
       apply: ${{ inputs.apply }}
 
+  update-reporting-v2:
+    uses: ./.github/workflows/configure-reporting-v2.yml
+    needs: [ publish-images, terraform ]
+    with:
+      image-tag: ${{ needs.publish-images.outputs.image-tag }}
+      environment: ${{ inputs.environment }}
+      apply: ${{ inputs.apply }}
+
   run-tests:
     if: ${{ inputs.apply }}
     needs: [

build/repositories.bzl

@@ -27,8 +27,8 @@ def wfa_measurement_system_repositories():
     wfa_repo_archive(
         name = "wfa_common_jvm",
         repo = "common-jvm",
-        sha256 = "c4536744dbe3d425f3713450a822af6b71de36799ce09da075f618a7a2fab03e",
-        version = "0.72.0",
+        sha256 = "4ccef195a7f4d03a78c5a538cf116dab55bf66a9d525c1c2f0d7c8b9a188da32",
+        version = "0.73.0",
     )
 
     wfa_repo_archive(

src/main/k8s/dev/reporting_v2_gke.cue

@@ -56,15 +56,15 @@ reporting: #Reporting & {
 	_internalApiTarget: certificateHost: "localhost"
 
 	_postgresConfig: {
-		iamUserLocal: "reporting-internal"
+		iamUserLocal: "reporting-v2-internal"
 		database:     "reporting-v2"
 	}
 
 	_verboseGrpcServerLogging: true
 
 	serviceAccounts: {
 		"\(#InternalServerServiceAccount)": #WorkloadIdentityServiceAccount & {
-			_iamServiceAccountName: "reporting-internal"
+			_iamServiceAccountName: "reporting-v2-internal"
 		}
 	}
 

src/main/k8s/reporting_v2.cue

@@ -18,7 +18,7 @@ package k8s
 	_verboseGrpcServerLogging: bool | *false
 	_verboseGrpcClientLogging: bool | *false
 
-	_reportSchedulingCronSchedule: *"30 6 * * *" // Daily at 6:30 AM
+	_reportSchedulingCronSchedule: string | *"30 6 * * *" // Daily at 6:30 AM
 
 	_postgresConfig: #PostgresConfig
 

src/main/terraform/gcloud/cmms/main.tf

@@ -19,10 +19,11 @@ terraform {
 }
 
 locals {
-  kingdom_cluster_name    = "kingdom"
-  duchy_names             = toset(["aggregator", "worker1", "worker2"])
-  reporting_cluster_name  = "reporting"
-  simulators_cluster_name = "simulators"
+  kingdom_cluster_name       = "kingdom"
+  duchy_names                = toset(["aggregator", "worker1", "worker2"])
+  reporting_cluster_name     = "reporting"
+  reporting_v2_cluster_name  = "reporting-v2"
+  simulators_cluster_name    = "simulators"
 
   cluster_location        = var.cluster_location == null ? data.google_client_config.default.zone : var.cluster_location
   key_ring_location       = var.key_ring_location == null ? data.google_client_config.default.region : var.key_ring_location

src/main/terraform/gcloud/cmms/reporting.tf

@@ -41,5 +41,7 @@ module "reporting_default_node_pool" {
 module "reporting" {
   source = "../modules/reporting"
 
-  postgres_instance = google_sql_database_instance.postgres
+  iam_service_account_name = "reporting-internal"
+  postgres_instance        = google_sql_database_instance.postgres
+  postgres_database_name   = "reporting"
 }

src/main/terraform/gcloud/cmms/reporting_v2.tf

@@ -0,0 +1,47 @@
+# Copyright 2024 The Cross-Media Measurement Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module "reporting_v2_cluster" {
+  source = "../modules/cluster"
+
+  name       = local.reporting_v2_cluster_name
+  location   = local.cluster_location
+  secret_key = module.common.cluster_secret_key
+}
+
+data "google_container_cluster" "reporting_v2" {
+  name     = local.reporting_v2_cluster_name
+  location = local.cluster_location
+
+  # Defer reading of cluster resource until it exists.
+  depends_on = [module.reporting_v2_cluster]
+}
+
+module "reporting_v2_default_node_pool" {
+  source = "../modules/node-pool"
+
+  name            = "default"
+  cluster         = data.google_container_cluster.reporting_v2
+  service_account = module.common.cluster_service_account
+  machine_type    = "e2-small"
+  max_node_count  = 8
+}
+
+module "reporting_v2" {
+  source = "../modules/reporting"
+
+  iam_service_account_name = "reporting-v2-internal"
+  postgres_instance        = google_sql_database_instance.postgres
+  postgres_database_name   = "reporting-v2"
+}

src/main/terraform/gcloud/modules/reporting/main.tf

@@ -25,7 +25,7 @@ module "reporting_internal" {
   source = "../workload-identity-user"
 
   k8s_service_account_name        = "internal-reporting-server"
-  iam_service_account_name        = "reporting-internal"
+  iam_service_account_name        = var.iam_service_account_name
   iam_service_account_description = "Reporting internal API server."
 }
 
@@ -48,7 +48,7 @@ resource "google_project_iam_member" "sql_client" {
 }
 
 resource "google_sql_database" "db" {
-  name     = "reporting"
+  name     = var.postgres_database_name
   instance = var.postgres_instance.name
 }
 
@@ -58,15 +58,3 @@ resource "postgresql_grant" "db" {
   object_type = "database"
   privileges  = local.all_db_privileges
 }
-
-resource "google_sql_database" "db2" {
-  name     = "reporting-v2"
-  instance = var.postgres_instance.name
-}
-
-resource "postgresql_grant" "db2" {
-  role        = google_sql_user.reporting_internal.name
-  database    = google_sql_database.db2.name
-  object_type = "database"
-  privileges  = local.all_db_privileges
-}

src/main/terraform/gcloud/modules/reporting/variables.tf

@@ -20,3 +20,16 @@ variable "postgres_instance" {
   })
   nullable = false
 }
+
+variable "postgres_database_name" {
+  description = "PostgreSQL `google_sql_database.name`."
+  type = string
+  nullable = false
+}
+
+variable "iam_service_account_name" {
+  description = "IAM `google_service_account.name`."
+  type = string
+  nullable = false
+}
+