Merge branch 'main' into tristanvuong-configure-reporting-workflow-to…

…-deploy-v2

WORKSPACE

@@ -116,6 +116,8 @@ MAVEN_ARTIFACTS_DICT = dict(common_jvm_maven_artifacts_dict().items() + {
     "software.amazon.awssdk:acmpca": AWS_JAVA_SDK_VERSION,
     "com.google.crypto.tink:tink-gcpkms": "1.9.0",
     "com.google.crypto.tink:tink-awskms": "1.9.1",
+    "com.google.apis:google-api-services-storage": "v1-rev20240105-2.0.0",
+    "com.google.apis:google-api-services-sqladmin": "v1-rev20240101-2.0.0",
 }.items())
 
 EXCLUDED_MAVEN_ARTIFACTS = [x for x in COMMON_JVM_EXCLUDED_ARTIFACTS if x != "org.slf4j:slf4j-log4j12"] + ["org.apache.beam:beam-sdks-java-io-kafka"]

build/repositories.bzl

@@ -41,8 +41,8 @@ def wfa_measurement_system_repositories():
     wfa_repo_archive(
         name = "wfa_measurement_proto",
         repo = "cross-media-measurement-api",
-        sha256 = "526ed0f2002a6a8c2e9082471c597e7688178918a38c2b024b9c3dace51884e1",
-        version = "0.55.0",
+        sha256 = "619e39693950807acd625f30e5f2110241cb1bd280e46553a09c3ed0177816a2",
+        version = "0.56.0",
     )
 
     wfa_repo_archive(

build/versions.bzl

@@ -21,6 +21,6 @@
 # * https://cloud.google.com/dataflow/docs/support/sdk-version-support-status#apache-beam-2.x-sdks
 # * https://beam.apache.org/documentation/runners/flink/#flink-version-compatibility
 # * https://docs.aws.amazon.com/kinesisanalytics/latest/java/earlier.html
-APACHE_BEAM_VERSION = "2.38.0"
+APACHE_BEAM_VERSION = "2.40.0"
 
 K8S_CLIENT_VERSION = "16.0.0"

src/main/cc/wfa/panelmatch/common/crypto/BUILD.bazel

@@ -88,6 +88,7 @@ cc_library(
     deps = [
         "@com_github_google_glog//:glog",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
         "@tink_cc//util:secret_data",
         "@wfa_common_cpp//src/main/cc/common_cpp/fingerprinters",
         "@wfa_common_cpp//src/main/cc/common_cpp/macros",

src/main/cc/wfa/panelmatch/common/crypto/peppered_fingerprinter.cc

@@ -17,6 +17,8 @@
 #include <memory>
 
 #include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "common_cpp/fingerprinters/fingerprinters.h"
 #include "glog/logging.h"
 #include "tink/util/secret_data.h"

src/main/k8s/base.cue

@@ -448,7 +448,10 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
 	_envVars:     #EnvVarMap
 	_javaOptions: #JavaOptions
 
-	_envVars: "JAVA_TOOL_OPTIONS": value: strings.Join(_javaOptions.options, " ")
+	_envVars: {
+		"JAVA_TOOL_OPTIONS": value: strings.Join(_javaOptions.options, " ")
+		"MALLOC_ARENA_MAX": value:  _ | *"2"
+	}
 
 	name:   string
 	image?: string

src/main/k8s/dev/synthetic_generator_edp_simulator_gke.cue

@@ -17,7 +17,7 @@ package k8s
 _resourceRequirements: ResourceRequirements=#ResourceRequirements & {
 	requests: {
 		cpu:    "500m"
-		memory: "288Mi"
+		memory: "768Mi"
 	}
 	limits: {
 		memory: ResourceRequirements.requests.memory
@@ -40,11 +40,11 @@ edp_simulators: {
 			_imageConfig: repoSuffix: "simulator/synthetic-generator-edp"
 			_additional_args: [
 				"--population-spec=\(_populationSpec)",
-				"--event-group-spec=\(EventGroupSpec)",
+				"--event-group-spec==\(EventGroupSpec)",
 			]
 			deployment: {
 				_container: {
-					_javaOptions: maxHeapSize: "96M"
+					_javaOptions: maxHeapSize: "512M"
 					resources: _resourceRequirements
 				}
 				spec: template: spec: {

src/main/k8s/local/edp_simulators.cue

@@ -71,7 +71,7 @@ edpSimulators: {
 			_kingdom_public_api_target: #KingdomPublicApiTarget
 			_additional_args: [
 				"--population-spec=\(_populationSpec)",
-				"--event-group-spec=\(edpConfig.eventGroupSpec)",
+				"--event-group-spec==\(edpConfig.eventGroupSpec)",
 			]
 
 			deployment: spec: template: spec: {

src/main/k8s/testing/secretfiles/llv2_protocol_config_config.textproto

@@ -1,9 +1,9 @@
-# proto-file: src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto
+# proto-file: wfa/measurement/internal/kingdom/protocol_config_config.proto
 # proto-message: Llv2ProtocolConfigConfig
 protocol_config {
   sketch_params {
-    decay_rate: 7.75
-    max_size: 100000
+    decay_rate: 5.6
+    max_size: 1000000
     sampling_indicator_size: 10000000
   }
   data_provider_noise {

src/main/kotlin/org/wfanet/measurement/api/v2alpha/ApiKeyKey.kt

@@ -26,7 +26,7 @@ data class ApiKeyKey(val measurementConsumerId: String, val apiKeyId: String) :
     return parser.assembleName(
       mapOf(
         IdVariable.MEASUREMENT_CONSUMER to measurementConsumerId,
-        IdVariable.API_KEY to apiKeyId
+        IdVariable.API_KEY to apiKeyId,
       )
     )
   }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DataProviderCertificateKey.kt

@@ -22,7 +22,7 @@ private val parser = ResourceNameParser("dataProviders/{data_provider}/certifica
 /** [ResourceKey] of a DataProvider Certificate. */
 data class DataProviderCertificateKey(
   val dataProviderId: String,
-  override val certificateId: String
+  override val certificateId: String,
 ) : CertificateKey {
   override val parentKey = DataProviderKey(dataProviderId)
 
@@ -39,7 +39,7 @@ data class DataProviderCertificateKey(
       return parser.parseIdVars(resourceName)?.let {
         DataProviderCertificateKey(
           it.getValue(IdVariable.DATA_PROVIDER),
-          it.getValue(IdVariable.CERTIFICATE)
+          it.getValue(IdVariable.CERTIFICATE),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DataProviderExchangeKey.kt

@@ -27,7 +27,7 @@ data class DataProviderExchangeKey(
   constructor(
     dataProviderId: String,
     recurringExchangeId: String,
-    exchangeId: String
+    exchangeId: String,
   ) : this(DataProviderRecurringExchangeKey(dataProviderId, recurringExchangeId), exchangeId)
 
   val dataProviderId: String
@@ -52,7 +52,7 @@ data class DataProviderExchangeKey(
       return DataProviderExchangeKey(
         idVars.getValue(IdVariable.DATA_PROVIDER),
         idVars.getValue(IdVariable.RECURRING_EXCHANGE),
-        idVars.getValue(IdVariable.EXCHANGE)
+        idVars.getValue(IdVariable.EXCHANGE),
       )
     }
   }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DataProviderExchangeStepAttemptKey.kt

@@ -32,7 +32,7 @@ data class DataProviderExchangeStepAttemptKey(
     exchangeStepAttemptId: String,
   ) : this(
     DataProviderExchangeStepKey(dataProviderId, recurringExchangeId, exchangeId, exchangeStepId),
-    exchangeStepAttemptId
+    exchangeStepAttemptId,
   )
 
   val dataProviderId: String

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DataProviderKey.kt

@@ -18,9 +18,7 @@ import org.wfanet.measurement.common.ResourceNameParser
 import org.wfanet.measurement.common.api.ResourceKey
 
 /** [DataProviderKey] of a Data Provider. */
-data class DataProviderKey(
-  val dataProviderId: String,
-) :
+data class DataProviderKey(val dataProviderId: String) :
   ResourceKey,
   CertificateParentKey,
   RecurringExchangeParentKey,

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DataProviderRecurringExchangeKey.kt

@@ -23,11 +23,11 @@ import org.wfanet.measurement.common.api.ResourceKey
 /** [ResourceKey] of a RecurringExchange with a DataProvider parent. */
 data class DataProviderRecurringExchangeKey(
   override val parentKey: DataProviderKey,
-  override val recurringExchangeId: String
+  override val recurringExchangeId: String,
 ) : RecurringExchangeKey, ChildResourceKey {
   constructor(
     dataProviderId: String,
-    recurringExchangeId: String
+    recurringExchangeId: String,
   ) : this(DataProviderKey(dataProviderId), recurringExchangeId)
 
   val dataProviderId: String
@@ -37,7 +37,7 @@ data class DataProviderRecurringExchangeKey(
     return parser.assembleName(
       mapOf(
         IdVariable.DATA_PROVIDER to dataProviderId,
-        IdVariable.RECURRING_EXCHANGE to recurringExchangeId
+        IdVariable.RECURRING_EXCHANGE to recurringExchangeId,
       )
     )
   }
@@ -50,7 +50,7 @@ data class DataProviderRecurringExchangeKey(
       val idVars: Map<IdVariable, String> = parser.parseIdVars(resourceName) ?: return null
       return DataProviderRecurringExchangeKey(
         idVars.getValue(IdVariable.DATA_PROVIDER),
-        idVars.getValue(IdVariable.RECURRING_EXCHANGE)
+        idVars.getValue(IdVariable.RECURRING_EXCHANGE),
       )
     }
   }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/DuchyCertificateKey.kt

@@ -20,10 +20,8 @@ import org.wfanet.measurement.common.api.ResourceKey
 private val parser = ResourceNameParser("duchies/{duchy}/certificates/{certificate}")
 
 /** [ResourceKey] of a Duchy Certificate. */
-data class DuchyCertificateKey(
-  val duchyId: String,
-  override val certificateId: String,
-) : CertificateKey {
+data class DuchyCertificateKey(val duchyId: String, override val certificateId: String) :
+  CertificateKey {
   override val parentKey = DuchyKey(duchyId)
 
   override fun toName(): String {

src/main/kotlin/org/wfanet/measurement/api/v2alpha/EventGroupMetadataDescriptorKey.kt

@@ -25,13 +25,13 @@ private val parser =
 /** [ResourceKey] of an EventGroupMetadataDescriptor. */
 data class EventGroupMetadataDescriptorKey(
   val dataProviderId: String,
-  val eventGroupMetadataDescriptorId: String
+  val eventGroupMetadataDescriptorId: String,
 ) : ResourceKey {
   override fun toName(): String {
     return parser.assembleName(
       mapOf(
         IdVariable.DATA_PROVIDER to dataProviderId,
-        IdVariable.EVENT_GROUP_METADATA_DESCRIPTOR to eventGroupMetadataDescriptorId
+        IdVariable.EVENT_GROUP_METADATA_DESCRIPTOR to eventGroupMetadataDescriptorId,
       )
     )
   }
@@ -43,7 +43,7 @@ data class EventGroupMetadataDescriptorKey(
       return parser.parseIdVars(resourceName)?.let {
         EventGroupMetadataDescriptorKey(
           it.getValue(IdVariable.DATA_PROVIDER),
-          it.getValue(IdVariable.EVENT_GROUP_METADATA_DESCRIPTOR)
+          it.getValue(IdVariable.EVENT_GROUP_METADATA_DESCRIPTOR),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/ExchangeKey.kt

@@ -43,7 +43,7 @@ data class CanonicalExchangeKey(
 ) : ExchangeKey {
   constructor(
     recurringExchangeId: String,
-    exchangeId: String
+    exchangeId: String,
   ) : this(CanonicalRecurringExchangeKey(recurringExchangeId), exchangeId)
 
   override fun toName(): String {
@@ -62,7 +62,7 @@ data class CanonicalExchangeKey(
       val idVars = parser.parseIdVars(resourceName) ?: return null
       return CanonicalExchangeKey(
         idVars.getValue(IdVariable.RECURRING_EXCHANGE),
-        idVars.getValue(IdVariable.EXCHANGE)
+        idVars.getValue(IdVariable.EXCHANGE),
       )
     }
   }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/ExchangeStepAttemptKey.kt

@@ -54,7 +54,7 @@ data class CanonicalExchangeStepAttemptKey(
     exchangeStepAttemptId: String,
   ) : this(
     CanonicalExchangeStepKey(recurringExchangeId, exchangeId, exchangeStepId),
-    exchangeStepAttemptId
+    exchangeStepAttemptId,
   )
 
   override fun toName(): String {
@@ -63,7 +63,7 @@ data class CanonicalExchangeStepAttemptKey(
         IdVariable.RECURRING_EXCHANGE to recurringExchangeId,
         IdVariable.EXCHANGE to exchangeId,
         IdVariable.EXCHANGE_STEP to exchangeStepId,
-        IdVariable.EXCHANGE_STEP_ATTEMPT to exchangeStepAttemptId
+        IdVariable.EXCHANGE_STEP_ATTEMPT to exchangeStepAttemptId,
       )
     )
   }
@@ -83,7 +83,7 @@ data class CanonicalExchangeStepAttemptKey(
           it.getValue(IdVariable.RECURRING_EXCHANGE),
           it.getValue(IdVariable.EXCHANGE),
           it.getValue(IdVariable.EXCHANGE_STEP),
-          it.getValue(IdVariable.EXCHANGE_STEP_ATTEMPT)
+          it.getValue(IdVariable.EXCHANGE_STEP_ATTEMPT),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/ExchangeStepKey.kt

@@ -42,20 +42,20 @@ sealed interface ExchangeStepKey : ChildResourceKey {
 /** Canonical [ResourceKey] of an ExchangeStep */
 data class CanonicalExchangeStepKey(
   override val parentKey: CanonicalExchangeKey,
-  override val exchangeStepId: String
+  override val exchangeStepId: String,
 ) : ExchangeStepKey {
   constructor(
     recurringExchangeId: String,
     exchangeId: String,
-    exchangeStepId: String
+    exchangeStepId: String,
   ) : this(CanonicalExchangeKey(recurringExchangeId, exchangeId), exchangeStepId)
 
   override fun toName(): String {
     return parser.assembleName(
       mapOf(
         IdVariable.RECURRING_EXCHANGE to recurringExchangeId,
         IdVariable.EXCHANGE to exchangeId,
-        IdVariable.EXCHANGE_STEP to exchangeStepId
+        IdVariable.EXCHANGE_STEP to exchangeStepId,
       )
     )
   }
@@ -72,7 +72,7 @@ data class CanonicalExchangeStepKey(
         CanonicalExchangeStepKey(
           it.getValue(IdVariable.RECURRING_EXCHANGE),
           it.getValue(IdVariable.EXCHANGE),
-          it.getValue(IdVariable.EXCHANGE_STEP)
+          it.getValue(IdVariable.EXCHANGE_STEP),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/MeasurementConsumerCertificateKey.kt

@@ -23,15 +23,15 @@ private val parser =
 /** [ResourceKey] of a MeasurementConsumer Certificate. */
 data class MeasurementConsumerCertificateKey(
   val measurementConsumerId: String,
-  override val certificateId: String
+  override val certificateId: String,
 ) : CertificateKey {
   override val parentKey = MeasurementConsumerKey(measurementConsumerId)
 
   override fun toName(): String {
     return parser.assembleName(
       mapOf(
         IdVariable.MEASUREMENT_CONSUMER to measurementConsumerId,
-        IdVariable.CERTIFICATE to certificateId
+        IdVariable.CERTIFICATE to certificateId,
       )
     )
   }
@@ -43,7 +43,7 @@ data class MeasurementConsumerCertificateKey(
       return parser.parseIdVars(resourceName)?.let {
         MeasurementConsumerCertificateKey(
           it.getValue(IdVariable.MEASUREMENT_CONSUMER),
-          it.getValue(IdVariable.CERTIFICATE)
+          it.getValue(IdVariable.CERTIFICATE),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/MeasurementConsumerEventGroupKey.kt

@@ -23,13 +23,13 @@ import org.wfanet.measurement.common.api.ResourceKey
 /** [ResourceKey] of an EventGroup with a MeasurementConsumer as the parent. */
 data class MeasurementConsumerEventGroupKey(
   val measurementConsumerId: String,
-  val eventGroupId: String
+  val eventGroupId: String,
 ) : ChildResourceKey {
   override fun toName(): String {
     return parser.assembleName(
       mapOf(
         IdVariable.MEASUREMENT_CONSUMER to measurementConsumerId,
-        IdVariable.EVENT_GROUP to eventGroupId
+        IdVariable.EVENT_GROUP to eventGroupId,
       )
     )
   }
@@ -46,7 +46,7 @@ data class MeasurementConsumerEventGroupKey(
       return parser.parseIdVars(resourceName)?.let {
         MeasurementConsumerEventGroupKey(
           it.getValue(IdVariable.MEASUREMENT_CONSUMER),
-          it.getValue(IdVariable.EVENT_GROUP)
+          it.getValue(IdVariable.EVENT_GROUP),
         )
       }
     }

src/main/kotlin/org/wfanet/measurement/api/v2alpha/MeasurementConsumerKey.kt

@@ -18,9 +18,8 @@ import org.wfanet.measurement.common.ResourceNameParser
 import org.wfanet.measurement.common.api.ResourceKey
 
 /** [ResourceKey] of a Measurement Consumer. */
-data class MeasurementConsumerKey(
-  val measurementConsumerId: String,
-) : ResourceKey, CertificateParentKey, PublicKeyParentKey {
+data class MeasurementConsumerKey(val measurementConsumerId: String) :
+  ResourceKey, CertificateParentKey, PublicKeyParentKey {
   override fun toName(): String {
     return parser.assembleName(mapOf(IdVariable.MEASUREMENT_CONSUMER to measurementConsumerId))
   }