diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 47f68bb6..c1f0c10d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.19 + go-version: 1.21 - name: Build go run: go build -mod=mod -v ./... diff --git a/java/dp_java_deps_preload.bzl b/java/dp_java_deps_preload.bzl index 40c0ae64..c942c93e 100644 --- a/java/dp_java_deps_preload.bzl +++ b/java/dp_java_deps_preload.bzl @@ -7,8 +7,8 @@ def dp_java_deps_prework(): This must be called before the rest of the dependencies are loaded. """ - RULES_JVM_EXTERNAL_TAG = "5.1" - RULES_JVM_EXTERNAL_SHA = "8c3b207722e5f97f1c83311582a6c11df99226e65e2471086e296561e57cc954" + RULES_JVM_EXTERNAL_TAG = "6.0" + RULES_JVM_EXTERNAL_SHA = "85fd6bad58ac76cc3a27c8e051e4255ff9ccd8c92ba879670d195622e7c0a9b7" BAZEL_COMMON_TAG = "a9e1d8efd54cbf27249695b23775b75ca65bb59d" BAZEL_COMMON_SHA = "17ea98149586dff60aa741c67fbd9a010fbb1507df90e741c50403bf5228bea3" http_archive( diff --git a/java/maven_install.json b/java/maven_install.json index d709250b..cc2393dd 100644 --- a/java/maven_install.json +++ b/java/maven_install.json @@ -1,6 +1,6 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -131666651, + "__INPUT_ARTIFACTS_HASH": -131668409, "__RESOLVED_ARTIFACTS_HASH": -1554454602, "artifacts": { "com.google.auto.value:auto-value": { diff --git a/privacy-on-beam/pbeam/count_test.go b/privacy-on-beam/pbeam/count_test.go index 7f1d09c8..45623d52 100644 --- a/privacy-on-beam/pbeam/count_test.go +++ b/privacy-on-beam/pbeam/count_test.go @@ -65,9 +65,7 @@ func TestCountNoNoise(t *testing.T) { pcol := MakePrivate(s, col, NewPrivacySpec(2*epsilon, delta)) got := Count(s, pcol, CountParams{MaxValue: 2, MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountNoNoise: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountNoNoise: Count(%v) = %v, expected %v: %v", col, got, want, err) } @@ -112,9 +110,7 @@ func TestCountWithPartitionsNoNoise(t *testing.T) { got := Count(s, pcol, countParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountWithPartitionsNoNoise in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountWithPartitionsNoNoise in-memory=%t: Count(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -381,9 +377,7 @@ func TestCountCrossPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountCrossPartitionContributionBounding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountCrossPartitionContributionBounding: Metric(%v) = %v, expected elements to sum to 150: %v", col, got, err) } @@ -430,9 +424,7 @@ func TestCountWithPartitionsCrossPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountWithPartitionsCrossPartitionContributionBounding in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountWithPartitionsCrossPartitionContributionBounding in-memory=%t: Metric(%v) = %v, expected elements to sum to 150: %v", tc.inMemory, col, got, err) } @@ -872,9 +864,7 @@ func TestCountNoNoiseTemp(t *testing.T) { pcol := MakePrivate(s, col, spec) got := Count(s, pcol, CountParams{MaxValue: 2, MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountNoNoiseTemp: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountNoNoiseTemp: Count(%v) = %v, expected %v: %v", col, got, want, err) } @@ -923,9 +913,7 @@ func TestCountWithPartitionsNoNoiseTemp(t *testing.T) { got := Count(s, pcol, countParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountWithPartitionsNoNoiseTemp in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountWithPartitionsNoNoiseTemp in-memory=%t: Count(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -966,9 +954,7 @@ func TestCountPreThresholding(t *testing.T) { pcol := MakePrivate(s, col, spec) got := Count(s, pcol, CountParams{MaxValue: int64(l1Sensitivity), MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestCountPreThresholding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestCountPreThresholding: Count(%v) = %v, expected %v: %v", col, got, want, err) } diff --git a/privacy-on-beam/pbeam/distinct_id_test.go b/privacy-on-beam/pbeam/distinct_id_test.go index 65463697..eefa1975 100644 --- a/privacy-on-beam/pbeam/distinct_id_test.go +++ b/privacy-on-beam/pbeam/distinct_id_test.go @@ -88,9 +88,7 @@ func TestDistinctPrivacyIDNoNoise(t *testing.T) { pcol := MakePrivate(s, col, NewPrivacySpec(epsilon, delta)) got := DistinctPrivacyID(s, pcol, DistinctPrivacyIDParams{MaxPartitionsContributed: 4, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDNoNoise: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDNoNoise: DistinctPrivacyID(%v) = %v, expected %v: %v", col, got, want, err) } @@ -141,9 +139,7 @@ func TestDistinctPrivacyIDWithPartitionsNoNoise(t *testing.T) { distinctPrivacyIDParams := DistinctPrivacyIDParams{MaxPartitionsContributed: 4, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := DistinctPrivacyID(s, pcol, distinctPrivacyIDParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDWithPartitionsNoNoise in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDWithPartitionsNoNoise in-memory=%t: DistinctPrivacyID(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -426,9 +422,7 @@ func TestDistinctPrivacyIDCrossPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDCrossPartitionContributionBounding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDCrossPartitionContributionBounding: DistinctPrivacyID(%v) = %v, expected elements to sum to 150: %v", col, got, err) } @@ -476,9 +470,7 @@ func TestDistinctPrivacyIDWithPartitionsCrossPartitionContributionBounding(t *te sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDWithPartitionsCrossPartitionContributionBounding in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDWithPartitionsCrossPartitionContributionBounding in-memory=%t: DistinctPrivacyID(%v) = %v, expected elements to sum to 150: %v", tc.inMemory, col, got, err) } @@ -579,9 +571,7 @@ func TestDistinctPrivacyIDOptimizedContrib(t *testing.T) { pcol := MakePrivate(s, col, NewPrivacySpec(epsilon, delta)) got := DistinctPrivacyID(s, pcol, DistinctPrivacyIDParams{MaxPartitionsContributed: 4, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDOptimizedContrib: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDOptimizedContrib: DistinctPrivacyID(%v) = %v, expected %v: %v", col, got, want, err) } @@ -1116,9 +1106,7 @@ func TestDistinctPrivacyIDNoNoiseTemp(t *testing.T) { pcol := MakePrivate(s, col, spec) got := DistinctPrivacyID(s, pcol, DistinctPrivacyIDParams{MaxPartitionsContributed: 4, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDNoNoiseTemp: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDNoNoiseTemp: DistinctPrivacyID(%v) = %v, expected %v: %v", col, got, want, err) } @@ -1172,9 +1160,7 @@ func TestDistinctPrivacyIDWithPartitionsNoNoiseTemp(t *testing.T) { distinctPrivacyIDParams := DistinctPrivacyIDParams{MaxPartitionsContributed: 4, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := DistinctPrivacyID(s, pcol, distinctPrivacyIDParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDWithPartitionsNoNoiseTemp in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDWithPartitionsNoNoiseTemp in-memory=%t: DistinctPrivacyID(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -1214,9 +1200,7 @@ func TestDistinctPrivacyIDPreThresholding(t *testing.T) { pcol := MakePrivate(s, col, spec) got := DistinctPrivacyID(s, pcol, DistinctPrivacyIDParams{MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPrivacyIDPreThresholding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPrivacyIDPreThresholding: Count(%v) = %v, expected %v: %v", col, got, want, err) } diff --git a/privacy-on-beam/pbeam/distinct_per_key_test.go b/privacy-on-beam/pbeam/distinct_per_key_test.go index 3024eed4..60eb2151 100644 --- a/privacy-on-beam/pbeam/distinct_per_key_test.go +++ b/privacy-on-beam/pbeam/distinct_per_key_test.go @@ -71,9 +71,7 @@ func TestDistinctPerKeyNoNoise(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 3, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 2}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyNoNoise: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyNoNoise: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -187,9 +185,7 @@ func TestDistinctPerKeyPerKeyCrossPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyCrossPartitionContributionBounding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyCrossPartitionContributionBounding: DistinctPerKey(%v) = %v, expected elements to sum to 150: %v", col, got, err) } @@ -235,9 +231,7 @@ func TestDistinctPerKeyWithPartitionsCrossPartitionContributionBounding(t *testi got = beam.AddFixedKey(s, maxOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("ApproxEqualsKVInt64 in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey with partitions in-memory=%t did not bound cross-partition contributions correctly: %v", tc.inMemory, err) } @@ -293,9 +287,7 @@ func TestDistinctPerKeyWithPartitionsPerPartitionContributionBounding(t *testing pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 3, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 2}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("ApproxEqualsKVInt64Slice in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey with partitions in-memory=%t did not bound cross-partition contributions correctly: %v", tc.inMemory, err) } @@ -337,9 +329,7 @@ func TestDistinctPerKeyCrossPartitionContributionBounding_IsAppliedBeforeDedupli pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 1}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyPerKeyCrossPartitionContributionBounding_IsAppliedBeforeDeduplication: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyPerKeyCrossPartitionContributionBounding_IsAppliedBeforeDeduplication: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -386,9 +376,7 @@ func TestDistinctPerKeyPerPartitionContributionBounding(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 3, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 2}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyPerPartitionContributionBounding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyPerPartitionContributionBounding: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -426,9 +414,7 @@ func TestDistinctPerKeyPerPartitionContributionBounding_IsAppliedBeforeDeduplica pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 1}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyPerPartitionContributionBounding_IsAppliedBeforeDeduplication: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyPerPartitionContributionBounding_IsAppliedBeforeDeduplication: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -539,9 +525,7 @@ func TestDistinctPerKeyThresholdsOnPrivacyIDs(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 1}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyNoNoise: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyNoNoise: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -756,9 +740,7 @@ func TestDistinctPerKeyNoNoiseTemp(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 3, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 2}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyNoNoiseTemp: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyNoNoiseTemp: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -789,10 +771,10 @@ func TestDistinctPerKeyWithPartitionNoNoiseTemp(t *testing.T) { } else { publicPartitions = beam.CreateList(s, publicParitionsSlice) } - // We use ε=50, δ=0 and l1Sensitivity=2. + // We use ε=50, δ=0 and l1Sensitivity=1. // We have 2 partitions. So, to get an overall flakiness of 10⁻²³, // we need to have each partition pass with 1-10⁻²⁴ probability (k=24) - epsilon, k, liSensitivity := 50.0, 24.0, 2.0 + epsilon, k, l1Sensitivity := 50.0, 24.0, 1.0 spec, err := NewPrivacySpecTemp(PrivacySpecParams{AggregationEpsilon: epsilon}) if err != nil { t.Fatalf("TestDistinctPerKeyWithPartitionNoNoiseTemp: %v", err) @@ -802,9 +784,7 @@ func TestDistinctPerKeyWithPartitionNoNoiseTemp(t *testing.T) { DistinctPerKeyParams := DistinctPerKeyParams{MaxPartitionsContributed: 1, MaxContributionsPerPartition: 1, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := DistinctPerKey(s, pcol, DistinctPerKeyParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, liSensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyWithPartitionNoNoiseTemp in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyWithPartitionNoNoiseTemp in-memory=%t: Count(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -844,9 +824,7 @@ func TestDistinctPerKeyPreThresholding(t *testing.T) { got := DistinctPerKey(s, pcol, DistinctPerKeyParams{MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, MaxContributionsPerPartition: 2}) // Assert - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestDistinctPerKeyPreThresholding: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestDistinctPerKeyPreThresholding: DistinctPerKey(%v) = %v, expected %v: %v", col, got, want, err) } diff --git a/privacy-on-beam/pbeam/mean_test.go b/privacy-on-beam/pbeam/mean_test.go index 16e51d71..5a334bc3 100644 --- a/privacy-on-beam/pbeam/mean_test.go +++ b/privacy-on-beam/pbeam/mean_test.go @@ -569,9 +569,7 @@ func TestMeanPerKeyNoNoiseFloat(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyNoNoiseFloat: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -623,9 +621,7 @@ func TestMeanPerKeyNoNoiseInt(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyNoNoiseInt: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -721,9 +717,7 @@ func TestMeanPerKeyWithPartitionsNoNoiseFloat(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean test case=%+v: got error %v", tc, err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64 test case=%+v: got error %v", tc, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyWithPartitionsNoNoiseFloat test case=%+v: MeanPerKey(%v) = %v, want %v, error %v", tc, col, got, want, err) } @@ -823,9 +817,7 @@ func TestMeanPerKeyWithPartitionsNoNoiseInt(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: test case=%+v got error %v", tc, err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64 test case=%+v: got error %v", tc, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyWithPartitionsNoNoiseInt test case=%+v: MeanPerKey(%v) = %v, want %v, error %v", tc, col, got, want, err) } @@ -881,9 +873,7 @@ func TestMeanPerKeyCountsPrivacyUnitIDsWithMultipleContributionsCorrectly(t *tes if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("MeanPerKey: for %v got %v, want %v, error %v", col, got, want, err) } @@ -1018,9 +1008,7 @@ func TestMeanKeyNegativeBounds(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyNegativeBounds: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -1086,9 +1074,7 @@ func TestMeanPerKeyCrossPartitionContributionBounding(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance1+tolerance2); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance1+tolerance2) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyCrossPartitionContributionBounding: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -1146,9 +1132,7 @@ func TestMeanPerKeyPerPartitionContributionBounding(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyPerPartitionContributionBounding: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -1351,9 +1335,7 @@ func TestMeanPerKeyWithPartitionsCrossPartitionContributionBounding(t *testing.T if err != nil { t.Fatalf("LaplaceToleranceForMean in-memory=%t: got error %v", tc.inMemory, err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance1+tolerance2); err != nil { - t.Fatalf("ApproxEqualsKVFloat64 in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance1+tolerance2) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyWithPartitionsPerPartitionContributionBounding in-memory=%t: MeanPerKey(%v) = %v, want %v, error %v", tc.inMemory, col, got, want, err) } @@ -1448,9 +1430,7 @@ func TestMeanPerKeyWithEmptyPartitionsNoNoise(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean test case=%+v: got error %v", tc, err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64 test case=%+v: got error %v", tc, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyWithEmptyPartitionsNoNoise test case=%+v: MeanPerKey(%v) = %v, want %v, error %v", tc, col, got, want, err) } @@ -1934,9 +1914,7 @@ func TestMeanPerKeyNoNoiseFloatTemp(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: got error %v", err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: got error %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyNoNoiseFloatTemp: MeanPerKey(%v) = %v, want %v, error %v", col, got, want, err) } @@ -2039,9 +2017,7 @@ func TestMeanPerKeyWithPartitionsNoNoiseIntTemp(t *testing.T) { if err != nil { t.Fatalf("LaplaceToleranceForMean: test case=%+v got error %v", tc, err) } - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64 test case=%+v: got error %v", tc, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("TestMeanPerKeyWithPartitionsNoNoiseIntTemp test case=%+v: MeanPerKey(%v) = %v, want %v, error %v", tc, col, got, want, err) } diff --git a/privacy-on-beam/pbeam/pbeam_test.go b/privacy-on-beam/pbeam/pbeam_test.go index 64eb530a..32871cfc 100644 --- a/privacy-on-beam/pbeam/pbeam_test.go +++ b/privacy-on-beam/pbeam/pbeam_test.go @@ -563,18 +563,13 @@ func TestDropKey(t *testing.T) { pcol = DropKey(s, pcol) // Assert that adding a test key of 0 and removing it is a no-op. - if err := testutils.EqualsKVInt(s, pcol.col, colKV); err != nil { - t.Fatalf("DropKey() did not work as expected: %v", err) - } + testutils.EqualsKVInt(t, s, pcol.col, colKV) publicPartitionsSlice := []int{100} publicPartitions := beam.CreateList(s, publicPartitionsSlice) got := Count(s, pcol, CountParams{MaxValue: 1, MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("Count() is different than expected: %v", err) - } - + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Error(err) } @@ -600,19 +595,14 @@ func TestDropValue(t *testing.T) { pcol = DropValue(s, pcol) // Assert that adding a test value of 0 and removing it is a no-op. - if err := testutils.EqualsKVInt(s, pcol.col, colKV); err != nil { - t.Fatalf("DropValue() did not work as expected: %v", err) - } + testutils.EqualsKVInt(t, s, pcol.col, colKV) publicPartitionsSlice := []int{100} publicPartitions := beam.CreateList(s, publicPartitionsSlice) got := Count(s, pcol, CountParams{MaxValue: 1, MaxPartitionsContributed: 1, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("Count() is different than expected: %v", err) - } - + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Error(err) } diff --git a/privacy-on-beam/pbeam/pbeamtest/pbeamtest_test.go b/privacy-on-beam/pbeam/pbeamtest/pbeamtest_test.go index f2d6b3f7..e586aa9e 100644 --- a/privacy-on-beam/pbeam/pbeamtest/pbeamtest_test.go +++ b/privacy-on-beam/pbeam/pbeamtest/pbeamtest_test.go @@ -89,9 +89,7 @@ func TestDistinctPrivacyIDTestMode(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPrivacyID: %s did not bound contributions correctly: %v", tc.desc, err) } @@ -152,9 +150,7 @@ func TestDistinctPrivacyIDWithPartitionsTestMode(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPrivacyID: %s with partitions did not bound contributions correctly: %v", tc.desc, err) } @@ -200,9 +196,7 @@ func TestDistinctPrivacyIDWithPartitionsTestModeAddsEmptyPartitions(t *testing.T NoiseKind: pbeam.LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPrivacyID: %s with partitions did not add empty partitions: %v", tc.desc, err) } @@ -265,9 +259,7 @@ func TestCountTestMode(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("Count: %s did not bound contributions correctly: %v", tc.desc, err) } @@ -334,9 +326,7 @@ func TestCountWithPartitionsTestMode(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("Count: %s with partitions did not bound contributions correctly: %v", tc.desc, err) } @@ -380,9 +370,7 @@ func TestCountWithPartitionsTestModeAddsEmptyPartitions(t *testing.T) { NoiseKind: pbeam.LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("Count: %s with partitions did not add empty partitions: %v", tc.desc, err) } @@ -449,9 +437,7 @@ func TestSumPerKeyTestModeInt(t *testing.T) { sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s did not bound contributions correctly for ints: %v", tc.desc, err) } @@ -521,9 +507,7 @@ func TestSumPerKeyWithPartitionsTestModeInt(t *testing.T) { sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s with partitions did not bound contributions correctly for ints: %v", tc.desc, err) } @@ -569,9 +553,7 @@ func TestSumPerKeyWithPartitionsTestModeAddsEmptyPartitionsInt(t *testing.T) { NoiseKind: pbeam.LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s with partitions did not add empty partitions for ints: %v", tc.desc, err) } @@ -638,9 +620,7 @@ func TestSumPerKeyTestModeFloat(t *testing.T) { sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.EqualsKVFloat64(s, got, want); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.EqualsKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s did not bound contributions correctly for floats: %v", tc.desc, err) } @@ -710,9 +690,7 @@ func TestSumPerKeyWithPartitionsTestModeFloat(t *testing.T) { sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.EqualsKVFloat64(s, got, want); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.EqualsKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s with partitions did not bound contributions correctly for floats: %v", tc.desc, err) } @@ -757,9 +735,7 @@ func TestSumPerKeyWithPartitionsTestModeAddsEmptyPartitionsFloat(t *testing.T) { NoiseKind: pbeam.LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.EqualsKVFloat64(s, got, want); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.EqualsKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("SumPerKey: %s with partitions did not add empty partitions for floats: %v", tc.desc, err) } @@ -819,9 +795,7 @@ func TestMeanPerKeyTestModeCrossPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, means) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.EqualsKVFloat64(s, got, want); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.EqualsKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("MeanPerKey: %s did not do cross-partition contribution bounding correctly: %v", tc.desc, err) } @@ -884,9 +858,7 @@ func TestMeanPerKeyTestModePerPartitionContributionBounding(t *testing.T) { NoiseKind: pbeam.LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) tolerance := 1e-10 // Using a small tolerance to make up for the rounding errors due to summation & division. - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("MeanPerKey: %s did not do per-partition contribution bounding correctly: %v", tc.desc, err) } @@ -947,9 +919,7 @@ func TestMeanPerKeyWithPartitionsTestModeCrossPartitionContributionBounding(t *t sumOverPartitions := stats.Sum(s, means) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.EqualsKVFloat64(s, got, want); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.EqualsKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("MeanPerKey: %s with partitions did not do cross-partition contribution bounding correctly: %v", tc.desc, err) } @@ -1018,9 +988,7 @@ func TestMeanPerKeyWithPartitionsTestModePerPartitionContributionBoundingAddsEmp PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) tolerance := 1e-10 // Using a small tolerance to make up for the rounding errors due to summation & division. - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("EqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("MeanPerKey: %s with partitions did not do per-partition contribution bounding correctly or added empty partitions: %v", tc.desc, err) } @@ -1101,9 +1069,7 @@ func TestQuantilesPerKeyTestModeCrossPartitionContributionBounding(t *testing.T) want = beam.ParDo(s, testutils.PairIF64ToKV, want) // Tolerance is multiplied by 2 because we sum over 2 partitions. tolerance := QuantilesTolerance(tc.minValue, tc.maxValue) * 2 - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey: %s did not bound cross-partition contributions correctly: %v", tc.desc, err) } @@ -1186,9 +1152,7 @@ func TestQuantilesPerKeyWithPartitionsTestModeCrossPartitionContributionBounding want = beam.ParDo(s, testutils.PairIF64ToKV, want) // Tolerance is multiplied by 2 because we sum over 2 partitions. tolerance := QuantilesTolerance(tc.minValue, tc.maxValue) * 2 - if err := testutils.ApproxEqualsKVFloat64(s, got, want, tolerance); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, tolerance) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey: %s with partitions did not bound cross-partition contributions correctly: %v", tc.desc, err) } @@ -1257,9 +1221,7 @@ func TestQuantilesPerKeyTestModePerPartitionContributionBounding(t *testing.T) { got = beam.ParDo(s, testutils.DereferenceFloat64Slice, got) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, QuantilesTolerance(tc.minValue, tc.maxValue)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, QuantilesTolerance(tc.minValue, tc.maxValue)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey: %s did not do per-partition contribution bounding correctly: %v", tc.desc, err) } @@ -1330,9 +1292,7 @@ func TestQuantilesPerKeyWithPartitionsTestModePerPartitionContributionBounding(t got = beam.ParDo(s, testutils.DereferenceFloat64Slice, got) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, QuantilesTolerance(tc.minValue, tc.maxValue)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, QuantilesTolerance(tc.minValue, tc.maxValue)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey: %s with partitions did not do per-partition contribution bounding correctly: %v", tc.desc, err) } @@ -1384,9 +1344,7 @@ func TestQuantilesPerKeyWithPartitionsAppliesPublicPartitions(t *testing.T) { }) want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey: %s did not apply public partitions correctly: %v", tc.desc, err) } @@ -1545,9 +1503,7 @@ func TestDistinctPerKeyTestModeCrossPartitionContributionBounding(t *testing.T) sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey: %s did not bound cross-partition contributions correctly: %v", tc.desc, err) } @@ -1605,9 +1561,7 @@ func TestDistinctPerKeyTestModePerPartitionContributionBounding(t *testing.T) { sumOverPartitions := stats.Sum(s, sums) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey: %s did not bound per-partition contributions correctly: %v", tc.desc, err) } @@ -1670,9 +1624,7 @@ func TestDistinctPerKeyWithPartitionsTestModeCrossPartitionContributionBounding( sumOverPartitions := stats.Sum(s, means) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey: %s with partitions did not bound contribution bounding correctly: %v", tc.desc, err) } @@ -1717,9 +1669,7 @@ func TestDistinctPerKeyWithPartitionsTestModeEmptyPartitionsInt(t *testing.T) { NoiseKind: pbeam.LaplaceNoise{}, PublicPartitions: publicPartitions}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.EqualsKVInt64(s, got, want); err != nil { - t.Fatalf("EqualsKVInt64: %v", err) - } + testutils.EqualsKVInt64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("DistinctPerKey: %s did not apply public partitions correctly: %v", tc.desc, err) } diff --git a/privacy-on-beam/pbeam/public_partitions_test.go b/privacy-on-beam/pbeam/public_partitions_test.go index ea26b011..42d54966 100644 --- a/privacy-on-beam/pbeam/public_partitions_test.go +++ b/privacy-on-beam/pbeam/public_partitions_test.go @@ -77,20 +77,19 @@ func TestPartitionMapFnMergeAccumulators(t *testing.T) { func TestDropNonPublicPartitionsVFn(t *testing.T) { pairs := testutils.ConcatenatePairs( testutils.MakePairsWithFixedV(7, 0), - testutils.MakePairsWithFixedV(52, 1), - testutils.MakePairsWithFixedV(99, 2), - testutils.MakePairsWithFixedV(10, 3), + testutils.MakePairsWithFixedVStartingFromKey(7, 10, 1), + testutils.MakePairsWithFixedVStartingFromKey(17, 83, 2), + testutils.MakePairsWithFixedVStartingFromKey(100, 10, 3), ) // Keep partitions 0, 2; // drop partitions 1, 3. result := testutils.ConcatenatePairs( testutils.MakePairsWithFixedV(7, 0), - testutils.MakePairsWithFixedV(99, 2), - testutils.MakePairsWithFixedV(99, 2), + testutils.MakePairsWithFixedVStartingFromKey(17, 83, 2), ) - _, s, col, want := ptest.CreateList2(pairs, result) + p, s, col, want := ptest.CreateList2(pairs, result) want = beam.ParDo(s, testutils.PairToKV, want) col = beam.ParDo(s, testutils.PairToKV, col) partitions := []int{0, 2} @@ -99,8 +98,9 @@ func TestDropNonPublicPartitionsVFn(t *testing.T) { epsilon, delta := 50.0, 1e-200 pcol := MakePrivate(s, col, NewPrivacySpec(epsilon, delta)) got := dropNonPublicPartitionsVFn(s, partitionsCol, pcol) - if err := testutils.EqualsKVInt(s, got, want); err != nil { - t.Fatalf("dropNonPublicPartitionsVFn: for %v got: %v, want %v", col, got, want) + testutils.EqualsKVInt(t, s, got, want) + if err := ptest.Run(p); err != nil { + t.Errorf("DropNonPublicPartitionsVFn did not drop non public partitions as expected: %v", err) } } @@ -108,18 +108,18 @@ func TestDropNonPublicPartitionsVFn(t *testing.T) { // are dropped (tests function used for sum and mean). func TestDropNonPublicPartitionsKVFn(t *testing.T) { triples := testutils.ConcatenateTriplesWithIntValue( - testutils.MakeSampleTripleWithIntValue(7, 0), - testutils.MakeSampleTripleWithIntValue(58, 1), - testutils.MakeSampleTripleWithIntValue(99, 2), - testutils.MakeSampleTripleWithIntValue(45, 100), - testutils.MakeSampleTripleWithIntValue(20, 33)) + testutils.MakeTripleWithIntValueStartingFromKey(0, 7, 0, 0), + testutils.MakeTripleWithIntValueStartingFromKey(7, 3, 1, 0), + testutils.MakeTripleWithIntValueStartingFromKey(10, 90, 2, 0), + testutils.MakeTripleWithIntValueStartingFromKey(100, 100, 11, 0), + testutils.MakeTripleWithIntValueStartingFromKey(200, 5, 12, 0)) // Keep partitions 0, 2. // Drop partitions 1, 33, 100. result := testutils.ConcatenateTriplesWithIntValue( - testutils.MakeSampleTripleWithIntValue(7, 0), - testutils.MakeSampleTripleWithIntValue(99, 2)) + testutils.MakeTripleWithIntValueStartingFromKey(0, 7, 0, 0), + testutils.MakeTripleWithIntValueStartingFromKey(10, 90, 2, 0)) - _, s, col, col2 := ptest.CreateList2(triples, result) + p, s, col, col2 := ptest.CreateList2(triples, result) // Doesn't matter that the values 3, 4, 5, 6, 9, 10 // are in the partitions PCollection because we are // just dropping the values that are in our original PCollection @@ -141,8 +141,9 @@ func TestDropNonPublicPartitionsKVFn(t *testing.T) { want := pcol2.col want = beam.SwapKV(s, want) - if err := testutils.EqualsKVInt(s, got, want); err != nil { - t.Fatalf("dropPublicPartitionsKVFn: for %v got: %v, want %v", col, got, want) + testutils.EqualsKVInt(t, s, got, want) + if err := ptest.Run(p); err != nil { + t.Errorf("TestDropNonPublicPartitionsKVFn did not drop non public partitions as expected: %v", err) } } @@ -162,7 +163,7 @@ func TestDropNonPublicPartitionsFloat(t *testing.T) { result := testutils.ConcatenateTriplesWithFloatValue( testutils.MakeTripleWithFloatValue(7, 0, 2.0)) - _, s, col, col2 := ptest.CreateList2(triples, result) + p, s, col, col2 := ptest.CreateList2(triples, result) // Doesn't matter that the values 2, 3, 4, 5, 6, 7 are in the partitions PCollection. // We are just dropping the values that are in our original PCollection that are not in @@ -184,7 +185,8 @@ func TestDropNonPublicPartitionsFloat(t *testing.T) { want := pcol2.col want = beam.SwapKV(s, want) - if err := testutils.EqualsKVInt(s, got, want); err != nil { - t.Fatalf("DropNonPublicPartitionsFloat: for %v got: %v, want %v", col, got, want) + testutils.EqualsKVInt(t, s, got, want) + if err := ptest.Run(p); err != nil { + t.Errorf("TestDropNonPublicPartitionsFloat did not drop non public partitions as expected: %v", err) } } diff --git a/privacy-on-beam/pbeam/quantiles_test.go b/privacy-on-beam/pbeam/quantiles_test.go index 7b0ec238..b8d88578 100644 --- a/privacy-on-beam/pbeam/quantiles_test.go +++ b/privacy-on-beam/pbeam/quantiles_test.go @@ -442,9 +442,7 @@ func TestQuantilesPerKeyAddsNoise(t *testing.T) { got1 = beam.ParDo(s, testutils.DereferenceFloat64Slice, got1) got2 = beam.ParDo(s, testutils.DereferenceFloat64Slice, got2) - if err := testutils.NotEqualsFloat64(s, got1, got2); err != nil { - t.Fatalf("NotEqualsFloat64: got error %v", err) - } + testutils.NotEqualsFloat64(t, s, got1, got2) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey didn't add any noise with float inputs and %s Noise: %v", tc.name, err) } @@ -525,9 +523,7 @@ func TestQuantilesWithPartitionsPerKeyAddsNoise(t *testing.T) { got1 = beam.ParDo(s, testutils.DereferenceFloat64Slice, got1) got2 = beam.ParDo(s, testutils.DereferenceFloat64Slice, got2) - if err := testutils.NotEqualsFloat64(s, got1, got2); err != nil { - t.Fatalf("NotEqualsFloat64: got error %v", err) - } + testutils.NotEqualsFloat64(t, s, got1, got2) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions %s didn't add any noise: %v", tc.desc, err) } @@ -568,9 +564,7 @@ func TestQuantilesPerKeyNoNoise(t *testing.T) { // Assert want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey did not return approximate quantile: %v", err) } @@ -625,9 +619,7 @@ func TestQuantilesPerKeyWithPartitionsNoNoise(t *testing.T) { // Assert want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions in-memory=%t did not return approximate quantile: %v", tc.inMemory, err) } @@ -828,9 +820,7 @@ func TestQuantilesPerKeyCrossPartitionContributionBounding(t *testing.T) { got = beam.AddFixedKey(s, maxOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.LessThanOrEqualToKVFloat64(s, got, want); err != nil { - t.Fatalf("LessThanOrEqualToKVFloat64: got error %v", err) - } + testutils.LessThanOrEqualToKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey did not bound cross-partition contributions correctly: %v", err) } @@ -896,9 +886,7 @@ func TestQuantilesPerKeyWithPartitionsCrossPartitionContributionBounding(t *test got = beam.AddFixedKey(s, maxOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.LessThanOrEqualToKVFloat64(s, got, want); err != nil { - t.Fatalf("LessThanOrEqualToKVFloat64 in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.LessThanOrEqualToKVFloat64(t, s, got, want) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions in-memory=%t did not bound cross-partition contributions correctly: %v", tc.inMemory, err) } @@ -940,9 +928,7 @@ func TestQuantilesPerKeyPerPartitionContributionBounding(t *testing.T) { }) want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey did not bound cross-partition contributions correctly: %v", err) } @@ -999,9 +985,7 @@ func TestQuantilesPerKeyWithPartitionsPerPartitionContributionBounding(t *testin got := QuantilesPerKey(s, pcol, quantilesParams) want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions in-memory=%t did not bound cross-partition contributions correctly: %v", tc.inMemory, err) } @@ -1039,9 +1023,7 @@ func TestQuantilesPerKeyAppliesClamping(t *testing.T) { }) want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey did not clamp input values: %v", err) } @@ -1094,9 +1076,7 @@ func TestQuantilesPerKeyWithPartitionsAppliesClamping(t *testing.T) { got := QuantilesPerKey(s, pcol, quantilesParams) want = beam.ParDo(s, testutils.PairIF64SliceToKV, want) - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions in-memory=%t did not clamp input values: %v", tc.inMemory, err) } @@ -1638,9 +1618,7 @@ func TestQuantilesPerKeyNoNoiseTemp(t *testing.T) { }) // Assert - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey did not return approximate quantile: %v", err) } @@ -1699,9 +1677,7 @@ func TestQuantilesPerKeyWithPartitionsNoNoiseTemp(t *testing.T) { got := QuantilesPerKey(s, pcol, quantilesParams) // Assert - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice in-memory=%t: got error %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("QuantilesPerKey with partitions in-memory=%t did not return approximate quantile: %v", tc.inMemory, err) } @@ -1751,9 +1727,7 @@ func TestQuantilesPerKeyPreThresholding(t *testing.T) { }) // Assert - if err := testutils.ApproxEqualsKVFloat64Slice(s, got, want, testutils.QuantilesTolerance(lower, upper)); err != nil { - t.Fatalf("ApproxEqualsKVFloat64Slice: got error %v", err) - } + testutils.ApproxEqualsKVFloat64Slice(t, s, got, want, testutils.QuantilesTolerance(lower, upper)) if err := ptest.Run(p); err != nil { t.Errorf("TestQuantilesPerKeyPreThresholding: %v", err) } diff --git a/privacy-on-beam/pbeam/sum_test.go b/privacy-on-beam/pbeam/sum_test.go index f6b7e65e..367acce2 100644 --- a/privacy-on-beam/pbeam/sum_test.go +++ b/privacy-on-beam/pbeam/sum_test.go @@ -60,9 +60,7 @@ func TestSumPerKeyNoNoiseInt(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 3, MinValue: 0.0, MaxValue: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyNoNoiseInt: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyNoNoiseInt: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -155,9 +153,7 @@ func TestSumPerKeyWithPartitionsNoNoiseInt(t *testing.T) { sumParams := SumParams{MaxPartitionsContributed: 3, MinValue: tc.minValue, MaxValue: tc.maxValue, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := SumPerKey(s, pcol, sumParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsNoNoiseInt test case=+%v: %v", tc, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsNoNoiseInt test case=+%v: SumPerKey(%v) = %v, expected %v: %v", tc, col, got, want, err) } @@ -185,9 +181,7 @@ func TestSumPerKeyNegativeBoundsInt(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 2, MinValue: -3, MaxValue: -2, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyNegativeBoundsInt: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyNegativeBoundsInt: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -229,9 +223,7 @@ func TestSumPerKeyWithPartitionsNegativeBoundsInt(t *testing.T) { sumParams := SumParams{MaxPartitionsContributed: 2, MinValue: -3, MaxValue: -2, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := SumPerKey(s, pcol, sumParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsNegativeBoundsInt in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsNegativeBoundsInt in-memory=%t: SumPerKey(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -262,9 +254,7 @@ func TestSumPerKeyNoNoiseFloat(t *testing.T) { pcol = ParDo(s, testutils.TripleWithFloatValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 3, MinValue: 0.0, MaxValue: 1.0, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyNoNoiseFloat: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyNoNoiseFloat: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -350,9 +340,7 @@ func TestSumPerKeyWithPartitionsNoNoiseFloat(t *testing.T) { sumParams := SumParams{MaxPartitionsContributed: 3, MinValue: tc.minValue, MaxValue: tc.maxValue, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := SumPerKey(s, pcol, sumParams) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsNoNoiseFloat test case=%+v: %v", tc, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsNoNoiseFloat test case=%+v: SumPerKey(%v) = %v, expected %v: %v", tc, col, got, want, err) } @@ -380,9 +368,7 @@ func TestSumPerKeyNegativeBoundsFloat(t *testing.T) { pcol = ParDo(s, testutils.TripleWithFloatValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 2, MinValue: -3.0, MaxValue: -2.0, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyNegativeBoundsFloat: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyNegativeBoundsFloat: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -425,9 +411,7 @@ func TestSumPerKeyWithPartitionsNegativeBoundsFloat(t *testing.T) { sumParams := SumParams{MaxPartitionsContributed: 2, MinValue: -3.0, MaxValue: -2.0, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := SumPerKey(s, pcol, sumParams) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsNegativeBoundsFloat in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsNegativeBoundsFloat in-memory=%t: SumPerKey(%v) = %v, expected %v: %v", tc.inMemory, col, got, want, err) } @@ -692,9 +676,7 @@ func TestSumPerKeyCrossPartitionContributionBoundingInt(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyCrossPartitionContributionBoundingInt: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyCrossPartitionContributionBoundingInt: SumPerKey(%v) = %v, expected elements to sum to 150: %v", col, got, err) } @@ -743,9 +725,7 @@ func TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingInt(t *testing sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingInt in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingInt in-memory=%t: SumPerKey(%v) = %v, expected elements to sum to 150: %v", tc.inMemory, col, got, err) } @@ -779,9 +759,7 @@ func TestSumPerKeyCrossPartitionContributionBoundingFloat(t *testing.T) { sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyCrossPartitionContributionBoundingFloat: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyCrossPartitionContributionBoundingFloat: SumPerKey(%v) = %v, expected elements to sum to 150.0: %v", col, got, err) } @@ -831,9 +809,7 @@ func TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingFloat(t *testi sumOverPartitions := stats.Sum(s, counts) got = beam.AddFixedKey(s, sumOverPartitions) // Adds a fixed key of 0. want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingFloat in-memory=%t: %v", tc.inMemory, err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsCrossPartitionContributionBoundingFloat in-memory=%t: SumPerKey(%v) = %v, expected elements to sum to 150.0: %v", tc.inMemory, col, got, err) } @@ -866,9 +842,7 @@ func TestSumPerKeyPerPartitionContributionBoundingInt(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MinValue: 2, MaxValue: 3, MaxPartitionsContributed: 2, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyPerPartitionContributionBoundingInt: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyPerPartitionContributionBoundingInt: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -900,9 +874,7 @@ func TestSumPerKeyPerPartitionContributionBoundingFloat(t *testing.T) { pcol = ParDo(s, testutils.TripleWithFloatValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MinValue: 2.0, MaxValue: 3.0, MaxPartitionsContributed: 2, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairIF64ToKV, want) - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyPerPartitionContributionBoundingFloat: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyPerPartitionContributionBoundingFloat: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -1683,9 +1655,7 @@ func TestSumPerKeyNoNoiseIntTemp(t *testing.T) { pcol = ParDo(s, testutils.TripleWithIntValueToKV, pcol) got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 3, MinValue: 0.0, MaxValue: 1, NoiseKind: LaplaceNoise{}}) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyNoNoiseIntTemp: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyNoNoiseIntTemp: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -1784,9 +1754,7 @@ func TestSumPerKeyWithPartitionsNoNoiseIntTemp(t *testing.T) { sumParams := SumParams{MaxPartitionsContributed: 3, MinValue: tc.minValue, MaxValue: tc.maxValue, NoiseKind: LaplaceNoise{}, PublicPartitions: publicPartitions} got := SumPerKey(s, pcol, sumParams) want = beam.ParDo(s, testutils.PairII64ToKV, want) - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyWithPartitionsNoNoiseIntTemp test case=+%v: %v", tc, err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyWithPartitionsNoNoiseIntTemp test case=+%v: SumPerKey(%v) = %v, expected %v: %v", tc, col, got, want, err) } @@ -1827,9 +1795,7 @@ func TestSumPerKeyPreThresholdingInt(t *testing.T) { got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 1, MinValue: 0, MaxValue: 1, NoiseKind: LaplaceNoise{}}) // Assert - if err := testutils.ApproxEqualsKVInt64(s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyPreThresholdingInt: %v", err) - } + testutils.ApproxEqualsKVInt64(t, s, got, want, testutils.RoundedLaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyPreThresholdingInt: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } @@ -1868,9 +1834,7 @@ func TestSumPerKeyPreThresholdingFloat(t *testing.T) { got := SumPerKey(s, pcol, SumParams{MaxPartitionsContributed: 1, MinValue: 0, MaxValue: 1, NoiseKind: LaplaceNoise{}}) // Assert - if err := testutils.ApproxEqualsKVFloat64(s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)); err != nil { - t.Fatalf("TestSumPerKeyPreThresholdingFloat: %v", err) - } + testutils.ApproxEqualsKVFloat64(t, s, got, want, testutils.LaplaceTolerance(k, l1Sensitivity, epsilon)) if err := ptest.Run(p); err != nil { t.Errorf("TestSumPerKeyPreThresholdingFloat: SumPerKey(%v) = %v, expected %v: %v", col, got, want, err) } diff --git a/privacy-on-beam/pbeam/testutils/testutils.go b/privacy-on-beam/pbeam/testutils/testutils.go index 6706b69f..0496ee7a 100644 --- a/privacy-on-beam/pbeam/testutils/testutils.go +++ b/privacy-on-beam/pbeam/testutils/testutils.go @@ -24,6 +24,7 @@ import ( "math" "math/big" "reflect" + "testing" "github.com/google/differential-privacy/go/v2/dpagg" "github.com/google/differential-privacy/go/v2/noise" @@ -313,50 +314,49 @@ func ConcatenatePairs(slices ...[]PairII) []PairII { // EqualsKVInt checks that two PCollections col1 and col2 of type // are exactly equal. -func EqualsKVInt(s beam.Scope, col1, col2 beam.PCollection) error { +func EqualsKVInt(t *testing.T, s beam.Scope, col1, col2 beam.PCollection) { + t.Helper() wantV := reflect.TypeOf(int(0)) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("EqualsKVInt: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("EqualsKVInt: unexpected value type for col2: %v", err) } coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, diffIntFn, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportDiffs, combinedDiff) - return nil } // EqualsKVInt64 checks that two PCollections col1 and col2 of type // are exactly equal. Each key can only hold a single value. -func EqualsKVInt64(s beam.Scope, col1, col2 beam.PCollection) error { - return ApproxEqualsKVInt64(s, col1, col2, 0.0) +func EqualsKVInt64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection) { + ApproxEqualsKVInt64(t, s, col1, col2, 0.0) } // EqualsKVFloat64 checks that two PCollections col1 and col2 of type // are exactly equal. Each key can only hold a single value. -func EqualsKVFloat64(s beam.Scope, col1, col2 beam.PCollection) error { - return ApproxEqualsKVFloat64(s, col1, col2, 0.0) +func EqualsKVFloat64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection) { + ApproxEqualsKVFloat64(t, s, col1, col2, 0.0) } // NotEqualsFloat64 checks that two PCollections col1 and col2 of type // are different. Each key can only hold a single value. -func NotEqualsFloat64(s beam.Scope, col1, col2 beam.PCollection) error { +func NotEqualsFloat64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection) { + t.Helper() wantV := reflect.TypeOf(float64(0)) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("NotEqualsFloat64: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("NotEqualsFloat64: unexpected value type for col2: %v", err) } - coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, &diffFloat64Fn{Tolerance: 0.0}, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportEquals, combinedDiff) - return nil } // ApproxEqualsKVInt64 checks that two PCollections col1 and col2 of type @@ -364,20 +364,20 @@ func NotEqualsFloat64(s beam.Scope, col1, col2 beam.PCollection) error { // "the keys are the same in both col1 and col2, and the value associated with // key k in col1 is within the specified tolerance of the value associated with k // in col2". Each key can only hold a single value. -func ApproxEqualsKVInt64(s beam.Scope, col1, col2 beam.PCollection, tolerance float64) error { +func ApproxEqualsKVInt64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection, tolerance float64) { + t.Helper() wantV := reflect.TypeOf(int64(0)) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("ApproxEqualsKVInt64: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("ApproxEqualsKVInt64: unexpected value type for col2: %v", err) } coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, &diffInt64Fn{Tolerance: tolerance}, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportDiffs, combinedDiff) - return nil } // ApproxEqualsKVFloat64 checks that two PCollections col1 and col2 of type @@ -385,39 +385,39 @@ func ApproxEqualsKVInt64(s beam.Scope, col1, col2 beam.PCollection, tolerance fl // "the keys are the same in both col1 and col2, and the value associated with // key k in col1 is within the specified tolerance of the value associated with k // in col2". Each key can only hold a single value. -func ApproxEqualsKVFloat64(s beam.Scope, col1, col2 beam.PCollection, tolerance float64) error { +func ApproxEqualsKVFloat64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection, tolerance float64) { + t.Helper() wantV := reflect.TypeOf(float64(0)) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("ApproxEqualsKVFloat64: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("ApproxEqualsKVFloat64: unexpected value type for col2: %v", err) } coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, &diffFloat64Fn{Tolerance: tolerance}, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportDiffs, combinedDiff) - return nil } // LessThanOrEqualToKVFloat64 checks that for PCollections col1 and col2 of type // , for each key k, value corresponding to col1 is less than or equal // to the value corresponding in col2. Each key can only hold a single value. -func LessThanOrEqualToKVFloat64(s beam.Scope, col1, col2 beam.PCollection) error { +func LessThanOrEqualToKVFloat64(t *testing.T, s beam.Scope, col1, col2 beam.PCollection) { + t.Helper() wantV := reflect.TypeOf(float64(0)) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("LessThanOrEqualToKVFloat64: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("LessThanOrEqualToKVFloat64: unexpected value type for col2: %v", err) } coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, lessThanOrEqualTo, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportGreaterThan, combinedDiff) - return nil } // ApproxEqualsKVFloat64Slice checks that two PCollections col1 and col2 of type @@ -425,20 +425,20 @@ func LessThanOrEqualToKVFloat64(s beam.Scope, col1, col2 beam.PCollection) error // "the keys are the same in both col1 and col2, and each value in the slice // associated with key k in col1 is within the specified tolerance of each value // in the slice associated with k in col2". Each key can only hold a single slice. -func ApproxEqualsKVFloat64Slice(s beam.Scope, col1, col2 beam.PCollection, tolerance float64) error { +func ApproxEqualsKVFloat64Slice(t *testing.T, s beam.Scope, col1, col2 beam.PCollection, tolerance float64) { + t.Helper() wantV := reflect.TypeOf([]float64{0.0}) if err := checkValueType(col1, wantV); err != nil { - return fmt.Errorf("unexpected value type for col1: %v", err) + t.Fatalf("ApproxEqualsKVFloat64Slice: unexpected value type for col1: %v", err) } if err := checkValueType(col2, wantV); err != nil { - return fmt.Errorf("unexpected value type for col2: %v", err) + t.Fatalf("ApproxEqualsKVFloat64Slice: unexpected value type for col2: %v", err) } coGroupToValue := beam.CoGroupByKey(s, col1, col2) diffs := beam.ParDo(s, &diffFloat64SliceFn{Tolerance: tolerance}, coGroupToValue) combinedDiff := beam.Combine(s, combineDiffs, diffs) beam.ParDo0(s, reportDiffs, combinedDiff) - return nil } func reportEquals(diffs string) error { diff --git a/privacy-on-beam/pbeam/testutils/testutils_test.go b/privacy-on-beam/pbeam/testutils/testutils_test.go index 3a3c30ec..9049e100 100644 --- a/privacy-on-beam/pbeam/testutils/testutils_test.go +++ b/privacy-on-beam/pbeam/testutils/testutils_test.go @@ -21,8 +21,6 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" ) // Tests for the test helpers. @@ -76,9 +74,7 @@ func TestApproxEqualsKVInt64(t *testing.T) { p, s, col1, col2 := ptest.CreateList2(tc.values1, tc.values2) col1KV := beam.ParDo(s, PairII64ToKV, col1) col2KV := beam.ParDo(s, PairII64ToKV, col2) - if err := ApproxEqualsKVInt64(s, col1KV, col2KV, tolerance); err != nil { - t.Fatalf("TestApproxEqualsKVInt64: %v", err) - } + ApproxEqualsKVInt64(t, s, col1KV, col2KV, tolerance) if err := ptest.Run(p); (err != nil) != tc.wantErr { t.Errorf("TestApproxEqualsKVInt64 failed for %s: got=%v, wantErr=%v", tc.desc, err, tc.wantErr) } @@ -130,23 +126,13 @@ func TestApproxEqualsKVFloat64(t *testing.T) { p, s, col1, col2 := ptest.CreateList2(tc.values1, tc.values2) col1KV := beam.ParDo(s, PairIF64ToKV, col1) col2KV := beam.ParDo(s, PairIF64ToKV, col2) - if err := ApproxEqualsKVFloat64(s, col1KV, col2KV, tolerance); err != nil { - t.Fatalf("TestApproxEqualsKVFloat64: %v", err) - } + ApproxEqualsKVFloat64(t, s, col1KV, col2KV, tolerance) if err := ptest.Run(p); (err != nil) != tc.wantErr { t.Errorf("TestApproxEqualsKVFloat64 failed for %s: got=%v, wantErr=%v", tc.desc, err, tc.wantErr) } } } -func assertFloat64PtrHasApproxValue(t *testing.T, got *float64, wantValue, tolerance float64) { - if got == nil { - t.Errorf("got , want: %g", wantValue) - } else if diff := cmp.Diff(*got, wantValue, cmpopts.EquateApprox(0, tolerance)); diff != "" { - t.Errorf("got %g, want %g", *got, wantValue) - } -} - func TestCheckNumPartitionsFn(t *testing.T) { for _, tc := range []struct { desc string