Fix reporting integration tests and EDP simulator tests (#1423)

* Extracted the common code of sampling VIDs to a function `sampleVids`. Affected files: * src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt * src/main/kotlin/org/wfanet/measurement/loadtest/measurementconsumer/MeasurementConsumerSimulator.kt * src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt * Computed expected values by VIDs instead of hardcoded values in EDP simulator tests. * Calculated tolerance for result check using metric variance in reporting integration tests. * Implemented impression metric result check in the reporting integration tests. * Fixed bug in reporting integration tests. Bug lists: * Use one event group to compute expected value when there are two in the reporting set. * The expected value of unique reach in a report is logically incorrect. * Report with cumulative doesn't set cumulative to true. * Added direct reach-only test to src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAMeasurementIntegrationTest.kt
world-federation-of-advertisers · Jan 24, 2024 · f38a8c5 · f38a8c5
1 parent 895da04
commit f38a8c5
Show file tree

Hide file tree

Showing 11 changed files with 608 additions and 651 deletions.
diff --git a/...n/org/wfanet/measurement/integration/common/InProcessLifeOfAMeasurementIntegrationTest.kt b/...n/org/wfanet/measurement/integration/common/InProcessLifeOfAMeasurementIntegrationTest.kt
@@ -132,6 +132,13 @@ abstract class InProcessLifeOfAMeasurementIntegrationTest(
       mcSimulator.testDirectReachAndFrequency("1234")
     }
 
+  @Test
+  fun `create a direct reach-only measurement and check the result is equal to the expected result`() =
+    runBlocking {
+      // Use frontend simulator to create a direct reach-only measurement and verify its result.
+      mcSimulator.testDirectReachOnly("1234")
+    }
+
   @Test
   fun `create a reach-only measurement and check the result is equal to the expected result`() =
     runBlocking {

diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/BUILD.bazel
@@ -21,6 +21,7 @@ kt_jvm_library(
         "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_cmms_components",
         "//src/main/kotlin/org/wfanet/measurement/integration/common:synthetic_generation_specs",
         "//src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/identity:reporting_principal_identity",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/common:sample_vids",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:synthetic_generator_event_query",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/measurementconsumer:synthetic_generator_event_query",
         "//src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server:internal_reporting_server",

diff --git a/...anet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt b/...anet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt
diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/common/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/common/BUILD.bazel
@@ -2,10 +2,24 @@ load("@wfa_common_jvm//build/rules_kotlin:defs.bzl", "kt_jvm_library")
 
 package(
     default_testonly = True,
-    default_visibility = ["//src/main/kotlin/org/wfanet/measurement/loadtest:__subpackages__"],
+    default_visibility = [
+        "//src/main/kotlin/org/wfanet/measurement/integration/common/reporting:__subpackages__",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest:__subpackages__",
+        "//src/test/kotlin/org/wfanet/measurement/loadtest:__subpackages__",
+    ],
 )
 
 kt_jvm_library(
     name = "output",
     srcs = ["Output.kt"],
 )
+
+kt_jvm_library(
+    name = "sample_vids",
+    srcs = ["SampleVids.kt"],
+    deps = [
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/config:vid_sampling",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:event_query",
+        "//src/main/proto/wfa/measurement/api/v2alpha:event_group_kt_jvm_proto",
+    ],
+)
diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/common/SampleVids.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/common/SampleVids.kt
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 The Cross-Media Measurement Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.wfanet.measurement.loadtest.common
+
+import com.google.protobuf.Message
+import org.wfanet.measurement.loadtest.config.VidSampling
+import org.wfanet.measurement.loadtest.dataprovider.EventQuery
+
+fun sampleVids(
+  eventQuery: EventQuery<Message>,
+  eventGroupSpecs: Iterable<EventQuery.EventGroupSpec>,
+  vidSamplingIntervalStart: Float,
+  vidSamplingIntervalWidth: Float,
+): Iterable<Long> {
+  require(vidSamplingIntervalWidth > 0 && vidSamplingIntervalWidth <= 1.0) {
+    "Invalid vidSamplingIntervalWidth $vidSamplingIntervalWidth"
+  }
+  require(
+    vidSamplingIntervalStart < 1 &&
+      vidSamplingIntervalStart >= 0 &&
+      vidSamplingIntervalWidth > 0 &&
+      vidSamplingIntervalStart + vidSamplingIntervalWidth <= 1
+  ) {
+    "Invalid vidSamplingInterval: start = $vidSamplingIntervalStart, width = " +
+      "$vidSamplingIntervalWidth"
+  }
+
+  return eventGroupSpecs
+    .asSequence()
+    .flatMap { eventQuery.getUserVirtualIds(it) }
+    .filter { vid ->
+      VidSampling.sampler.vidIsInSamplingBucket(
+        vid,
+        vidSamplingIntervalStart,
+        vidSamplingIntervalWidth,
+      )
+    }
+    .asIterable()
+}
diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel
@@ -144,6 +144,7 @@ kt_jvm_library(
         "//src/main/kotlin/org/wfanet/measurement/eventdataprovider/noiser",
         "//src/main/kotlin/org/wfanet/measurement/eventdataprovider/privacybudgetmanagement:privacy_budget_manager",
         "//src/main/kotlin/org/wfanet/measurement/eventdataprovider/privacybudgetmanagement/api/v2alpha:privacy_query_mapper",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/common:sample_vids",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:privacy_budgets",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:test_identifiers",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:vid_sampling",

diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt
@@ -139,8 +139,8 @@ import org.wfanet.measurement.eventdataprovider.privacybudgetmanagement.Referenc
 import org.wfanet.measurement.eventdataprovider.privacybudgetmanagement.api.v2alpha.PrivacyQueryMapper.getDirectAcdpQuery
 import org.wfanet.measurement.eventdataprovider.privacybudgetmanagement.api.v2alpha.PrivacyQueryMapper.getDpQuery
 import org.wfanet.measurement.eventdataprovider.privacybudgetmanagement.api.v2alpha.PrivacyQueryMapper.getLiquidLegionsV2AcdpQuery
+import org.wfanet.measurement.loadtest.common.sampleVids
 import org.wfanet.measurement.loadtest.config.TestIdentifiers.SIMULATOR_EVENT_GROUP_REFERENCE_ID_PREFIX
-import org.wfanet.measurement.loadtest.config.VidSampling
 import org.wfanet.measurement.loadtest.dataprovider.MeasurementResults.computeImpression
 
 data class EdpData(
@@ -1184,32 +1184,8 @@ class EdpSimulator(
     eventGroupSpecs: Iterable<EventQuery.EventGroupSpec>,
     vidSamplingInterval: MeasurementSpec.VidSamplingInterval,
   ): Iterable<Long> {
-    val vidSamplingIntervalStart = vidSamplingInterval.start
-    val vidSamplingIntervalWidth = vidSamplingInterval.width
-
-    require(vidSamplingIntervalWidth > 0 && vidSamplingIntervalWidth <= 1.0) {
-      "Invalid vidSamplingIntervalWidth $vidSamplingIntervalWidth"
-    }
-    require(
-      vidSamplingIntervalStart < 1 &&
-        vidSamplingIntervalStart >= 0 &&
-        vidSamplingIntervalWidth > 0 &&
-        vidSamplingIntervalStart + vidSamplingIntervalWidth <= 1
-    ) {
-      "Invalid vidSamplingInterval: $vidSamplingInterval"
-    }
     return try {
-      eventGroupSpecs
-        .asSequence()
-        .flatMap { eventQuery.getUserVirtualIds(it) }
-        .filter { vid ->
-          VidSampling.sampler.vidIsInSamplingBucket(
-            vid,
-            vidSamplingIntervalStart,
-            vidSamplingIntervalWidth,
-          )
-        }
-        .asIterable()
+      sampleVids(eventQuery, eventGroupSpecs, vidSamplingInterval.start, vidSamplingInterval.width)
     } catch (e: EventFilterValidationException) {
       logger.log(
         Level.WARNING,

diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/measurementconsumer/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/measurementconsumer/BUILD.bazel
@@ -41,6 +41,7 @@ kt_jvm_library(
         "//src/main/kotlin/org/wfanet/measurement/api/v2alpha/testing",
         "//src/main/kotlin/org/wfanet/measurement/common/identity",
         "//src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha:api_key_authentication_server_interceptor",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/common:sample_vids",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:test_identifiers",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:vid_sampling",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:event_query",

diff --git a/...otlin/org/wfanet/measurement/loadtest/measurementconsumer/MeasurementConsumerSimulator.kt b/...otlin/org/wfanet/measurement/loadtest/measurementconsumer/MeasurementConsumerSimulator.kt
@@ -98,8 +98,8 @@ import org.wfanet.measurement.consent.client.measurementconsumer.signMeasurement
 import org.wfanet.measurement.consent.client.measurementconsumer.signRequisitionSpec
 import org.wfanet.measurement.consent.client.measurementconsumer.verifyResult
 import org.wfanet.measurement.eventdataprovider.noiser.DpParams as NoiserDpParams
+import org.wfanet.measurement.loadtest.common.sampleVids
 import org.wfanet.measurement.loadtest.config.TestIdentifiers
-import org.wfanet.measurement.loadtest.config.VidSampling
 import org.wfanet.measurement.loadtest.dataprovider.EventQuery
 import org.wfanet.measurement.loadtest.dataprovider.MeasurementResults
 import org.wfanet.measurement.measurementconsumer.stats.DeterministicMethodology
@@ -164,50 +164,51 @@ class MeasurementConsumerSimulator(
 
   private val MeasurementInfo.sampledVids: Sequence<Long>
     get() {
-      return requisitions.asSequence().flatMap {
-        val eventGroupsMap: Map<String, RequisitionSpec.EventGroupEntry.Value> =
-          it.requisitionSpec.eventGroupsMap
-        it.eventGroups.flatMap { eventGroup ->
-          sampleVids(
-            EventQuery.EventGroupSpec(eventGroup, eventGroupsMap.getValue(eventGroup.name)),
-            measurementSpec.vidSamplingInterval,
-          )
-        }
-      }
+      val eventGroupSpecs =
+        requisitions
+          .flatMap {
+            val eventGroupsMap: Map<String, RequisitionSpec.EventGroupEntry.Value> =
+              it.requisitionSpec.eventGroupsMap
+            it.eventGroups.map { eventGroup ->
+              EventQuery.EventGroupSpec(eventGroup, eventGroupsMap.getValue(eventGroup.name))
+            }
+          }
+          .asIterable()
+      return sampleVids(eventGroupSpecs, measurementSpec.vidSamplingInterval)
     }
 
   private fun MeasurementInfo.sampleVidsByDataProvider(
     targetDataProviderId: String
   ): Sequence<Long> {
-    return requisitions.asSequence().flatMap { requisitionInfo ->
-      val eventGroupsMap: Map<String, RequisitionSpec.EventGroupEntry.Value> =
-        requisitionInfo.requisitionSpec.eventGroupsMap
-
-      requisitionInfo.eventGroups
-        .filter { eventGroup ->
-          targetDataProviderId ==
-            requireNotNull(EventGroupKey.fromName(eventGroup.name)).dataProviderId
-        }
-        .flatMap { eventGroup ->
-          sampleVids(
-            EventQuery.EventGroupSpec(eventGroup, eventGroupsMap.getValue(eventGroup.name)),
-            measurementSpec.vidSamplingInterval,
-          )
+    val eventGroupSpecs =
+      requisitions
+        .flatMap { requisitionInfo ->
+          val eventGroupsMap: Map<String, RequisitionSpec.EventGroupEntry.Value> =
+            requisitionInfo.requisitionSpec.eventGroupsMap
+          requisitionInfo.eventGroups
+            .filter { eventGroup ->
+              targetDataProviderId ==
+                requireNotNull(EventGroupKey.fromName(eventGroup.name)).dataProviderId
+            }
+            .map { eventGroup ->
+              EventQuery.EventGroupSpec(eventGroup, eventGroupsMap.getValue(eventGroup.name))
+            }
         }
-    }
+        .asIterable()
+    return sampleVids(eventGroupSpecs, measurementSpec.vidSamplingInterval)
   }
 
   private fun sampleVids(
-    eventGroupSpec: EventQuery.EventGroupSpec,
+    eventGroupSpecs: Iterable<EventQuery.EventGroupSpec>,
     vidSamplingInterval: VidSamplingInterval,
   ): Sequence<Long> {
-    return eventQuery.getUserVirtualIds(eventGroupSpec).filter { vid ->
-      VidSampling.sampler.vidIsInSamplingBucket(
-        vid,
+    return sampleVids(
+        eventQuery,
+        eventGroupSpecs,
         vidSamplingInterval.start,
         vidSamplingInterval.width,
       )
-    }
+      .asSequence()
   }
 
   data class ExecutionResult(
@@ -350,7 +351,7 @@ class MeasurementConsumerSimulator(
   }
 
   /** A sequence of operations done in the simulator involving a direct reach measurement. */
-  suspend fun testDirectReach(runId: String) {
+  suspend fun testDirectReachOnly(runId: String) {
     // Create a new measurement on behalf of the measurement consumer.
     val measurementConsumer = getMeasurementConsumer(measurementConsumerData.name)
     val measurementInfo =
@@ -380,6 +381,7 @@ class MeasurementConsumerSimulator(
 
     assertThat(reachResult.reach.hasDeterministicCountDistinct()).isTrue()
     assertThat(reachResult.reach.noiseMechanism).isEqualTo(expectedDirectNoiseMechanism)
+    assertThat(reachResult.hasFrequency()).isFalse()
 
     logger.info("Direct reach result is equal to the expected result")
   }

diff --git a/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel
@@ -20,10 +20,12 @@ kt_jvm_library(
         "//src/main/kotlin/org/wfanet/measurement/eventdataprovider/privacybudgetmanagement/testing",
         "//src/main/kotlin/org/wfanet/measurement/integration/common:configs",
         "//src/main/kotlin/org/wfanet/measurement/integration/common:synthetic_generation_specs",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/common:sample_vids",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:event_group_metadata",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:test_identifiers",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:edp_simulator",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:in_memory_event_query",
+        "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:measurement_results",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:synthetic_generator_event_query",
         "//src/main/proto/wfa/measurement/api/v2alpha:protocol_config_kt_jvm_proto",
         "//src/main/proto/wfa/measurement/api/v2alpha/event_templates/testing:test_event_kt_jvm_proto",