From 6ec24b22cec2348964a2e0fc438c3fe1e5be5461 Mon Sep 17 00:00:00 2001 From: Differential Privacy Team Date: Mon, 21 Oct 2024 08:27:03 -0700 Subject: [PATCH] Cleanup and documentation changes C++: * cleanup code, refine comments * update absl dependency Java: * Load java rules from @rules_java Change-Id: I3141a3389af98ba221c13c230b36e9afb2420187 GitOrigin-RevId: cb0c561fbd518ac22686e23e7833ddf2c75e9468 --- README.md | 3 + cc/accounting/accountant.cc | 5 +- cc/accounting/accountant.h | 2 +- cc/accounting/accountant_test.cc | 4 +- cc/accounting/common/common.h | 2 +- cc/accounting/common/common_test.cc | 39 +++++++------ cc/accounting/convolution.cc | 3 +- cc/accounting/convolution.h | 2 +- cc/accounting/privacy_loss_distribution.cc | 3 +- cc/accounting/privacy_loss_distribution.h | 2 +- .../privacy_loss_distribution_test.cc | 4 +- cc/accounting/privacy_loss_mechanism.cc | 3 +- cc/accounting/privacy_loss_mechanism.h | 2 +- cc/algorithms/approx-bounds.h | 4 +- cc/algorithms/bounded-algorithm.h | 12 ++-- cc/algorithms/bounded-mean.h | 6 +- cc/algorithms/count.h | 2 +- cc/algorithms/numerical-mechanisms.cc | 2 +- cc/algorithms/numerical-mechanisms.h | 26 ++++----- cc/algorithms/partition-selection.h | 12 ++-- cc/algorithms/partition-selection_test.cc | 8 +-- cc/algorithms/quantile-tree.h | 9 +-- cc/algorithms/quantiles.h | 7 ++- cc/algorithms/util.cc | 55 +++++++++---------- cc/algorithms/util.h | 46 ++++++++-------- cc/algorithms/util_test.cc | 12 ++-- cc/cc_differential_privacy_deps.bzl | 12 ++-- examples/java/BUILD | 3 + .../google/privacy/differentialprivacy/BUILD | 3 + .../privacy/differentialprivacy/testing/BUILD | 3 + .../google/privacy/differentialprivacy/BUILD | 2 + .../differentialprivacy/statistical/BUILD | 2 + .../privacy/differentialprivacy/testing/BUILD | 2 + privacy-on-beam/pbeam/mean_test.go | 9 +++ proto/data.proto | 5 +- .../configs/dataset_generator_config.py | 4 +- 36 files changed, 180 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index 13c909b2..1c91d04d 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,9 @@ the contents of a dataset and/or its order. Our integer implementations are not subject to the vulnerabilities described in the paper (though note that Java does not have an integer implementation). +Please refer to our [attack model](common_docs/attack_model.md) to learn more +about how to use our libraries in a safe way. + ## Support We will continue to publish updates and improvements to the library. We are diff --git a/cc/accounting/accountant.cc b/cc/accounting/accountant.cc index d482a31a..ebcc0c32 100644 --- a/cc/accounting/accountant.cc +++ b/cc/accounting/accountant.cc @@ -14,6 +14,7 @@ #include "accounting/accountant.h" #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -28,14 +29,14 @@ namespace accounting { absl::StatusOr GetSmallestParameter(EpsilonDelta epsilon_delta, int num_queries, double sensitivity, NoiseFunction noise_function, - absl::optional upper_bound, + std::optional upper_bound, double tolerance) { BinarySearchParameters search_parameters = { .lower_bound = 0, .upper_bound = upper_bound.has_value() ? upper_bound.value() : 2 * num_queries * sensitivity / epsilon_delta.epsilon, - .initial_guess = absl::nullopt, + .initial_guess = std::nullopt, .tolerance = tolerance}; auto compute_delta = [noise_function, sensitivity, num_queries, diff --git a/cc/accounting/accountant.h b/cc/accounting/accountant.h index 5be857a6..1a65f881 100644 --- a/cc/accounting/accountant.h +++ b/cc/accounting/accountant.h @@ -34,7 +34,7 @@ using NoiseFunction = std::function( absl::StatusOr GetSmallestParameter(EpsilonDelta epsilon_delta, int num_queries, double sensitivity, NoiseFunction noise_function, - absl::optional upper_bound, + std::optional upper_bound, double tolerance = 1e-4); // Uses the optimal advanced composition theorem, Theorem 3.3 from the paper diff --git a/cc/accounting/accountant_test.cc b/cc/accounting/accountant_test.cc index 916f3e6f..e4f21b00 100644 --- a/cc/accounting/accountant_test.cc +++ b/cc/accounting/accountant_test.cc @@ -14,6 +14,8 @@ #include "accounting/accountant.h" +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/status/status.h" @@ -74,7 +76,7 @@ TEST_P(AccountantTest, Basic) { auto result = GetSmallestParameter(param.epsilon_delta, param.num_queries, param.sensitivity, param.noise_function, - absl::optional()); + std::optional()); EXPECT_OK(result.status()); EXPECT_NEAR(result.value(), param.expected_parameter, kMaxError); } diff --git a/cc/accounting/common/common.h b/cc/accounting/common/common.h index 866a1e13..c486dae4 100644 --- a/cc/accounting/common/common.h +++ b/cc/accounting/common/common.h @@ -54,7 +54,7 @@ struct BinarySearchParameters { // An initial guess to start the search with. Must be positive. When this // guess is close to the true value, it can help make the binary search // faster. - absl::optional initial_guess; + std::optional initial_guess; // An acceptable error on the returned value. double tolerance = 1e-7; // Whether the search is over integers. diff --git a/cc/accounting/common/common_test.cc b/cc/accounting/common/common_test.cc index 5b17eaf0..a9b7316b 100644 --- a/cc/accounting/common/common_test.cc +++ b/cc/accounting/common/common_test.cc @@ -14,6 +14,9 @@ #include "accounting/common/common.h" +#include +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/status/status.h" @@ -56,7 +59,7 @@ struct InverseMonotoneFunctionParam { double value; double lower_x; double upper_x; - absl::optional initial_guess; + std::optional initial_guess; bool increasing; double expected_x; bool discrete = false; @@ -73,7 +76,7 @@ INSTANTIATE_TEST_SUITE_P( .value = -4.5, .lower_x = 0, .upper_x = 10, - .initial_guess = absl::nullopt, + .initial_guess = std::nullopt, .increasing = false, .expected_x = 4.5}, // with initial guess @@ -94,27 +97,29 @@ INSTANTIATE_TEST_SUITE_P( .increasing = false, .expected_x = 5}, // increasing, without initial guess - InverseMonotoneFunctionParam{.func = [](double x) { return pow(x, 2); }, - .value = 25, - .lower_x = 0, - .upper_x = 10, - .initial_guess = absl::nullopt, - .increasing = true, - .expected_x = 5}, + InverseMonotoneFunctionParam{ + .func = [](double x) { return std::pow(x, 2); }, + .value = 25, + .lower_x = 0, + .upper_x = 10, + .initial_guess = std::nullopt, + .increasing = true, + .expected_x = 5}, // increasing, with initial guess - InverseMonotoneFunctionParam{.func = [](double x) { return pow(x, 2); }, - .value = 25, - .lower_x = 0, - .upper_x = 10, - .initial_guess = 2, - .increasing = true, - .expected_x = 5}, + InverseMonotoneFunctionParam{ + .func = [](double x) { return std::pow(x, 2); }, + .value = 25, + .lower_x = 0, + .upper_x = 10, + .initial_guess = 2, + .increasing = true, + .expected_x = 5}, // discrete InverseMonotoneFunctionParam{.func = [](double x) { return -x; }, .value = -4.5, .lower_x = 0, .upper_x = 10, - .initial_guess = absl::nullopt, + .initial_guess = std::nullopt, .increasing = false, .expected_x = 5, .discrete = true})); diff --git a/cc/accounting/convolution.cc b/cc/accounting/convolution.cc index 1a7ced3a..1fe9c112 100644 --- a/cc/accounting/convolution.cc +++ b/cc/accounting/convolution.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -121,7 +122,7 @@ ProbabilityMassFunction Convolve(const ProbabilityMassFunction& x, ConvolutionTruncationBounds ComputeConvolutionTruncationBounds( const UnpackedProbabilityMassFunction& x, int num_times, - double tail_mass_truncation, absl::optional> orders) { + double tail_mass_truncation, std::optional> orders) { const int size = x.items.size(); int64_t upper_bound = static_cast(size - 1) * num_times; int64_t lower_bound = 0; diff --git a/cc/accounting/convolution.h b/cc/accounting/convolution.h index f4c2eeb3..e5c83d26 100644 --- a/cc/accounting/convolution.h +++ b/cc/accounting/convolution.h @@ -69,7 +69,7 @@ struct ConvolutionTruncationBounds { ConvolutionTruncationBounds ComputeConvolutionTruncationBounds( const UnpackedProbabilityMassFunction& x, int num_times, double tail_mass_truncation = 0, - absl::optional> orders = {}); + std::optional> orders = {}); // Returns convolution of probability mass function with itself num_times. // Additional parameter: diff --git a/cc/accounting/privacy_loss_distribution.cc b/cc/accounting/privacy_loss_distribution.cc index 0141ba9a..6938e9fe 100644 --- a/cc/accounting/privacy_loss_distribution.cc +++ b/cc/accounting/privacy_loss_distribution.cc @@ -15,6 +15,7 @@ #include #include +#include #include #include "absl/status/status.h" @@ -230,7 +231,7 @@ PrivacyLossDistribution::CreateForGaussianMechanism( absl::StatusOr> PrivacyLossDistribution::CreateForDiscreteGaussianMechanism( double sigma, int sensitivity, EstimateType estimate_type, - double discretization_interval, absl::optional truncation_bound) { + double discretization_interval, std::optional truncation_bound) { ASSIGN_OR_RETURN(std::unique_ptr privacy_loss, DiscreteGaussianPrivacyLoss::Create(sigma, sensitivity, truncation_bound)); diff --git a/cc/accounting/privacy_loss_distribution.h b/cc/accounting/privacy_loss_distribution.h index cc23bf89..b7854c02 100644 --- a/cc/accounting/privacy_loss_distribution.h +++ b/cc/accounting/privacy_loss_distribution.h @@ -195,7 +195,7 @@ class PrivacyLossDistribution { double sigma, int sensitivity = 1, EstimateType estimate_type = EstimateType::kPessimistic, double discretization_interval = 1e-4, - absl::optional truncation_bound = absl::nullopt); + std::optional truncation_bound = std::nullopt); // Creates {@link PrivacyLossDistribution} from epsilon and delta parameters. // diff --git a/cc/accounting/privacy_loss_distribution_test.cc b/cc/accounting/privacy_loss_distribution_test.cc index 6a9f9e09..4ba86fb7 100644 --- a/cc/accounting/privacy_loss_distribution_test.cc +++ b/cc/accounting/privacy_loss_distribution_test.cc @@ -14,6 +14,8 @@ #include "accounting/privacy_loss_distribution.h" +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/status/statusor.h" @@ -860,7 +862,7 @@ struct DiscreteGaussianPrivacyLossDistributionParam { double discretization_interval; ProbabilityMassFunction expected_pmf; double expected_infinity_mass; - absl::optional truncation_bound = absl::nullopt; + std::optional truncation_bound = std::nullopt; }; class DiscreteGaussianPrivacyLossDistribution diff --git a/cc/accounting/privacy_loss_mechanism.cc b/cc/accounting/privacy_loss_mechanism.cc index f8e2f374..45d0893f 100644 --- a/cc/accounting/privacy_loss_mechanism.cc +++ b/cc/accounting/privacy_loss_mechanism.cc @@ -14,6 +14,7 @@ #include "accounting/privacy_loss_mechanism.h" #include +#include #include "base/status_macros.h" @@ -244,7 +245,7 @@ PrivacyLossTail DiscreteLaplacePrivacyLoss::PrivacyLossDistributionTail() } absl::StatusOr> DiscreteGaussianPrivacyLoss::Create(double sigma, int sensitivity, - absl::optional truncation_bound) { + std::optional truncation_bound) { if (sigma <= 0) { return absl::InvalidArgumentError("sigma should be positive."); } diff --git a/cc/accounting/privacy_loss_mechanism.h b/cc/accounting/privacy_loss_mechanism.h index 173825b4..8cc35328 100644 --- a/cc/accounting/privacy_loss_mechanism.h +++ b/cc/accounting/privacy_loss_mechanism.h @@ -312,7 +312,7 @@ class DiscreteGaussianPrivacyLoss : public AdditiveNoisePrivacyLoss { // outside of this range is at most 1e-30. static absl::StatusOr> Create( double sigma, int sensitivity, - absl::optional truncation_bound = absl::nullopt); + std::optional truncation_bound = std::nullopt); NoiseType Discrete() const override { return NoiseType::kDiscrete; } diff --git a/cc/algorithms/approx-bounds.h b/cc/algorithms/approx-bounds.h index 78547c7b..da343a71 100644 --- a/cc/algorithms/approx-bounds.h +++ b/cc/algorithms/approx-bounds.h @@ -811,13 +811,13 @@ class ApproxBounds::Builder { } } - absl::optional epsilon_; + std::optional epsilon_; int max_partitions_contributed_ = 1; int max_contributions_per_partition_ = 1; std::unique_ptr mechanism_builder_ = absl::make_unique(); - absl::optional threshold_; + std::optional threshold_; double scale_ = DefaultScaleForT(); double base_ = 2.0; double success_probability_ = 1 - std::pow(10, -9); diff --git a/cc/algorithms/bounded-algorithm.h b/cc/algorithms/bounded-algorithm.h index cf786bd4..a3b47f99 100644 --- a/cc/algorithms/bounded-algorithm.h +++ b/cc/algorithms/bounded-algorithm.h @@ -86,24 +86,24 @@ class BoundedAlgorithmBuilder : public AlgorithmBuilder { // automatically this will be the full epsilon - epsilon spent on that // ApproxBounds. If called before BoundsSetup this will always return the full // epsilon. - absl::optional GetRemainingEpsilon() { + std::optional GetRemainingEpsilon() { if (remaining_epsilon_.has_value()) { return remaining_epsilon_; } return AlgorithmBuilder::GetEpsilon(); } - absl::optional GetLower() const { return lower_; } - absl::optional GetUpper() const { return upper_; } + std::optional GetLower() const { return lower_; } + std::optional GetUpper() const { return upper_; } private: // Bounds are optional and do not need to be set. If they are not set, // automatic bounds will be determined. - absl::optional lower_; - absl::optional upper_; + std::optional lower_; + std::optional upper_; // Epsilon left over after creating an ApproxBounds. - absl::optional remaining_epsilon_; + std::optional remaining_epsilon_; // Common initialization and checks for building bounded algorithms. absl::StatusOr> BuildAlgorithm() final { diff --git a/cc/algorithms/bounded-mean.h b/cc/algorithms/bounded-mean.h index 5c1d9e28..05682553 100644 --- a/cc/algorithms/bounded-mean.h +++ b/cc/algorithms/bounded-mean.h @@ -533,10 +533,10 @@ class BoundedMean::Builder { } private: - absl::optional epsilon_; + std::optional epsilon_; double delta_ = 0; - absl::optional upper_; - absl::optional lower_; + std::optional upper_; + std::optional lower_; int max_partitions_contributed_ = 1; int max_contributions_per_partition_ = 1; std::unique_ptr mechanism_builder_ = diff --git a/cc/algorithms/count.h b/cc/algorithms/count.h index 9bc58445..f03c20c3 100644 --- a/cc/algorithms/count.h +++ b/cc/algorithms/count.h @@ -178,7 +178,7 @@ class Count::Builder { } private: - absl::optional epsilon_; + std::optional epsilon_; double delta_ = 0; int max_partitions_contributed_ = 1; int max_contributions_per_partition_ = 1; diff --git a/cc/algorithms/numerical-mechanisms.cc b/cc/algorithms/numerical-mechanisms.cc index 032c343d..7db1dd1d 100644 --- a/cc/algorithms/numerical-mechanisms.cc +++ b/cc/algorithms/numerical-mechanisms.cc @@ -245,7 +245,7 @@ GaussianMechanism::Builder::Build() { return result; } // Else construct from DP parameters. - absl::optional epsilon = GetEpsilon(); + std::optional epsilon = GetEpsilon(); RETURN_IF_ERROR(ValidateIsFiniteAndPositive(epsilon, "Epsilon")); RETURN_IF_ERROR(DeltaIsSetAndValid()); ASSIGN_OR_RETURN(double l2, CalculateL2Sensitivity()); diff --git a/cc/algorithms/numerical-mechanisms.h b/cc/algorithms/numerical-mechanisms.h index 6c60d9a0..c261c420 100644 --- a/cc/algorithms/numerical-mechanisms.h +++ b/cc/algorithms/numerical-mechanisms.h @@ -166,18 +166,16 @@ class NumericalMechanismBuilder { return absl::OkStatus(); } - absl::optional GetEpsilon() const { return epsilon_; } - absl::optional GetDelta() const { return delta_; } - absl::optional GetL0Sensitivity() const { return l0_sensitivity_; } - absl::optional GetLInfSensitivity() const { - return linf_sensitivity_; - } + std::optional GetEpsilon() const { return epsilon_; } + std::optional GetDelta() const { return delta_; } + std::optional GetL0Sensitivity() const { return l0_sensitivity_; } + std::optional GetLInfSensitivity() const { return linf_sensitivity_; } private: - absl::optional epsilon_; - absl::optional delta_; - absl::optional l0_sensitivity_; - absl::optional linf_sensitivity_; + std::optional epsilon_; + std::optional delta_; + std::optional l0_sensitivity_; + std::optional linf_sensitivity_; }; // Provides differential privacy by adding Laplace noise. This class also @@ -205,10 +203,10 @@ class LaplaceMechanism : public NumericalMechanism { } protected: - absl::optional GetL1Sensitivity() const { return l1_sensitivity_; } + std::optional GetL1Sensitivity() const { return l1_sensitivity_; } private: - absl::optional l1_sensitivity_; + std::optional l1_sensitivity_; }; ABSL_DEPRECATED( @@ -333,8 +331,8 @@ class GaussianMechanism : public NumericalMechanism { } protected: - absl::optional l2_sensitivity_; - absl::optional stddev_; + std::optional l2_sensitivity_; + std::optional stddev_; private: // Returns the l2 sensitivity when it has been set or returns an upper bound diff --git a/cc/algorithms/partition-selection.h b/cc/algorithms/partition-selection.h index 7a414904..e6808dd4 100644 --- a/cc/algorithms/partition-selection.h +++ b/cc/algorithms/partition-selection.h @@ -354,12 +354,12 @@ class LaplacePartitionSelection : public PartitionSelectionStrategy { return mechanism_->NoisedValueAboveThreshold(num_users, threshold_); } - absl::optional NoiseValueIfShouldKeep(double num_users) { + std::optional NoiseValueIfShouldKeep(double num_users) { if (num_users < GetPreThreshold()) { - return absl::optional(); + return std::optional(); } double noised_value = mechanism_->AddNoise(num_users); - return noised_value > threshold_ ? noised_value : absl::optional(); + return noised_value > threshold_ ? noised_value : std::optional(); } double ProbabilityOfKeep(double num_users) const override { @@ -529,12 +529,12 @@ class GaussianPartitionSelection : public PartitionSelectionStrategy { return mechanism_->NoisedValueAboveThreshold(num_users, threshold_); } - absl::optional NoiseValueIfShouldKeep(double num_users) { + std::optional NoiseValueIfShouldKeep(double num_users) { if (num_users < GetPreThreshold()) { - return absl::optional(); + return std::optional(); } double noised_value = mechanism_->AddNoise(num_users); - return noised_value > threshold_ ? noised_value : absl::optional(); + return noised_value > threshold_ ? noised_value : std::optional(); } double ProbabilityOfKeep(double num_users) const override { diff --git a/cc/algorithms/partition-selection_test.cc b/cc/algorithms/partition-selection_test.cc index a4e00b49..c587b94c 100644 --- a/cc/algorithms/partition-selection_test.cc +++ b/cc/algorithms/partition-selection_test.cc @@ -592,7 +592,7 @@ TEST(PartitionSelectionTest, struct CalculateDeltaTest { CalculateDeltaTest(double epsilon_in, double threshold_in, int64_t max_partitions_contributed_in, - absl::optional expected_delta_in, + std::optional expected_delta_in, double tolerance_in = kCalcDeltaTestDefaultTolerance) : epsilon(epsilon_in), threshold(threshold_in), @@ -604,7 +604,7 @@ struct CalculateDeltaTest { const double threshold; const int64_t max_partitions_contributed; // Missing implies an error is expected to be returned. - const absl::optional expected_delta; + const std::optional expected_delta; const double tolerance; }; @@ -763,7 +763,7 @@ TEST(PartitionSelectionTest, CalculateDeltaTests) { struct CalculateThresholdTest { CalculateThresholdTest( double epsilon_in, double delta_in, int64_t max_partitions_contributed_in, - absl::optional expected_threshold_in, + std::optional expected_threshold_in, double tolerance_in = kCalcThresholdTestDefaultTolerance) : epsilon(epsilon_in), delta(delta_in), @@ -775,7 +775,7 @@ struct CalculateThresholdTest { const double delta; const int64_t max_partitions_contributed; // Missing implies an error is expected to be returned. - const absl::optional expected_threshold; + const std::optional expected_threshold; const double tolerance; }; diff --git a/cc/algorithms/quantile-tree.h b/cc/algorithms/quantile-tree.h index 42bcb0b5..6e9cfebe 100644 --- a/cc/algorithms/quantile-tree.h +++ b/cc/algorithms/quantile-tree.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "absl/status/status.h" @@ -561,10 +562,10 @@ class QuantileTree::Builder { } private: - absl::optional tree_height_; - absl::optional branching_factor_; - absl::optional lower_; - absl::optional upper_; + std::optional tree_height_; + std::optional branching_factor_; + std::optional lower_; + std::optional upper_; }; } // namespace differential_privacy diff --git a/cc/algorithms/quantiles.h b/cc/algorithms/quantiles.h index 3d5313fe..119bb739 100644 --- a/cc/algorithms/quantiles.h +++ b/cc/algorithms/quantiles.h @@ -18,6 +18,7 @@ #define DIFFERENTIAL_PRIVACY_CPP_ALGORITHMS_QUANTILES_H_ #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -231,10 +232,10 @@ class Quantiles::Builder { } private: - absl::optional epsilon_; + std::optional epsilon_; double delta_ = 0; - absl::optional upper_; - absl::optional lower_; + std::optional upper_; + std::optional lower_; int max_partitions_contributed_ = 1; int max_contributions_per_partition_ = 1; std::unique_ptr mechanism_builder_ = diff --git a/cc/algorithms/util.cc b/cc/algorithms/util.cc index 8041e4f2..aadcdff0 100644 --- a/cc/algorithms/util.cc +++ b/cc/algorithms/util.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "absl/status/status.h" @@ -108,7 +109,7 @@ int64_t RoundToNearestInt64Multiple(int64_t n, int64_t base) { return n - remainder; } -absl::Status ValidateIsSet(absl::optional opt, absl::string_view name, +absl::Status ValidateIsSet(std::optional opt, absl::string_view name, absl::StatusCode error_code) { if (!opt.has_value()) { return absl::InvalidArgumentError(absl::StrCat(name, " must be set.")); @@ -122,7 +123,7 @@ absl::Status ValidateIsSet(absl::optional opt, absl::string_view name, return absl::OkStatus(); } -absl::Status ValidateIsPositive(absl::optional opt, +absl::Status ValidateIsPositive(std::optional opt, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); @@ -134,7 +135,7 @@ absl::Status ValidateIsPositive(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateIsNonNegative(absl::optional opt, +absl::Status ValidateIsNonNegative(std::optional opt, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); @@ -147,8 +148,7 @@ absl::Status ValidateIsNonNegative(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateIsFinite(absl::optional opt, - absl::string_view name, +absl::Status ValidateIsFinite(std::optional opt, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); double d = opt.value(); @@ -159,7 +159,7 @@ absl::Status ValidateIsFinite(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateIsFiniteAndPositive(absl::optional opt, +absl::Status ValidateIsFiniteAndPositive(std::optional opt, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); @@ -172,7 +172,7 @@ absl::Status ValidateIsFiniteAndPositive(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateIsFiniteAndNonNegative(absl::optional opt, +absl::Status ValidateIsFiniteAndNonNegative(std::optional opt, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); @@ -186,7 +186,7 @@ absl::Status ValidateIsFiniteAndNonNegative(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateIsInInclusiveInterval(absl::optional opt, +absl::Status ValidateIsInInclusiveInterval(std::optional opt, double lower_bound, double upper_bound, absl::string_view name, @@ -195,7 +195,7 @@ absl::Status ValidateIsInInclusiveInterval(absl::optional opt, error_code); } -absl::Status ValidateIsInExclusiveInterval(absl::optional opt, +absl::Status ValidateIsInExclusiveInterval(std::optional opt, double lower_bound, double upper_bound, absl::string_view name, @@ -204,8 +204,8 @@ absl::Status ValidateIsInExclusiveInterval(absl::optional opt, error_code); } -absl::Status ValidateIsLesserThan(absl::optional opt, - double upper_bound, absl::string_view name, +absl::Status ValidateIsLesserThan(std::optional opt, double upper_bound, + absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); if (opt.value() < upper_bound) { @@ -216,7 +216,7 @@ absl::Status ValidateIsLesserThan(absl::optional opt, ", but is ", opt.value(), ".")); } -absl::Status ValidateIsLesserThanOrEqualTo(absl::optional opt, +absl::Status ValidateIsLesserThanOrEqualTo(std::optional opt, double upper_bound, absl::string_view name, absl::StatusCode error_code) { @@ -229,7 +229,7 @@ absl::Status ValidateIsLesserThanOrEqualTo(absl::optional opt, upper_bound, ", but is ", opt.value(), ".")); } -absl::Status ValidateIsGreaterThan(absl::optional opt, +absl::Status ValidateIsGreaterThan(std::optional opt, double lower_bound, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); @@ -241,7 +241,7 @@ absl::Status ValidateIsGreaterThan(absl::optional opt, ", but is ", opt.value(), ".")); } -absl::Status ValidateIsGreaterThanOrEqualTo(absl::optional opt, +absl::Status ValidateIsGreaterThanOrEqualTo(std::optional opt, double lower_bound, absl::string_view name, absl::StatusCode error_code) { @@ -254,10 +254,9 @@ absl::Status ValidateIsGreaterThanOrEqualTo(absl::optional opt, lower_bound, ", but is ", opt.value(), ".")); } -absl::Status ValidateIsInInterval(absl::optional opt, - double lower_bound, double upper_bound, - bool include_lower, bool include_upper, - absl::string_view name, +absl::Status ValidateIsInInterval(std::optional opt, double lower_bound, + double upper_bound, bool include_lower, + bool include_upper, absl::string_view name, absl::StatusCode error_code) { RETURN_IF_ERROR(ValidateIsSet(opt, name, error_code)); double d = opt.value(); @@ -289,44 +288,44 @@ absl::Status ValidateIsInInterval(absl::optional opt, return absl::OkStatus(); } -absl::Status ValidateEpsilon(absl::optional epsilon) { +absl::Status ValidateEpsilon(std::optional epsilon) { return ValidateIsFiniteAndPositive(epsilon, "Epsilon"); } -absl::Status ValidateDelta(absl::optional delta) { +absl::Status ValidateDelta(std::optional delta) { return ValidateIsInInclusiveInterval(delta, 0, 1, "Delta"); } absl::Status ValidateMaxPartitionsContributed( - absl::optional max_partitions_contributed) { + std::optional max_partitions_contributed) { return ValidateIsPositive(max_partitions_contributed, "Maximum number of partitions that can be " "contributed to (i.e., L0 sensitivity)"); } absl::Status ValidateMaxContributionsPerPartition( - absl::optional max_contributions_per_partition) { + std::optional max_contributions_per_partition) { return ValidateIsPositive(max_contributions_per_partition, "Maximum number of contributions per partition"); } -absl::Status ValidateMaxContributions(absl::optional max_contributions) { +absl::Status ValidateMaxContributions(std::optional max_contributions) { return ValidateIsPositive( max_contributions, "Maximum number of contributions (i.e., L1 sensitivity)"); } -absl::Status ValidateTreeHeight(absl::optional tree_height) { +absl::Status ValidateTreeHeight(std::optional tree_height) { return ValidateIsGreaterThanOrEqualTo(tree_height, /*lower_bound=*/1, "Tree Height"); } -absl::Status ValidateBranchingFactor(absl::optional branching_factor) { +absl::Status ValidateBranchingFactor(std::optional branching_factor) { return ValidateIsGreaterThanOrEqualTo(branching_factor, /*lower_bound=*/2, "Branching Factor"); } -absl::Status ValidatePreThresholdOptional(absl::optional pre_threshold) { +absl::Status ValidatePreThresholdOptional(std::optional pre_threshold) { int v = pre_threshold.value_or(1); if (v <= 0) { return absl::Status( @@ -341,9 +340,9 @@ absl::Status ValidatePreThresholdOptional(absl::optional pre_threshold) { [[deprecated( "This validator is used for a class that is deprecated in favour of the " "pre_threshold attribute of other strategies classes.")]] absl::Status -ValidatePreThreshold(absl::optional pre_threshold) { +ValidatePreThreshold(std::optional pre_threshold) { return ValidateIsGreaterThan( - static_cast>(pre_threshold), /*lower_bound=*/0, + static_cast>(pre_threshold), /*lower_bound=*/0, "Pre Threshold"); } } // namespace differential_privacy diff --git a/cc/algorithms/util.h b/cc/algorithms/util.h index 2b631c29..39001ca6 100644 --- a/cc/algorithms/util.h +++ b/cc/algorithms/util.h @@ -423,42 +423,42 @@ std::string VectorToString(const std::vector& v) { // Otherwise, will return an `error_code` error that includes `name` in the // error message. absl::Status ValidateIsSet( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // positive. Otherwise, will return an `error_code` error status that includes // `name` in the error message. absl::Status ValidateIsPositive( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // non-negative. Otherwise, will return an `error_code` error status that // includes `name` in the error message. absl::Status ValidateIsNonNegative( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // finite. Otherwise, will return an `error_code` error status that includes // `name` in the error message. absl::Status ValidateIsFinite( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set, finite, // and positive. Otherwise, will return an `error_code` error status that // includes `name` in the error message. absl::Status ValidateIsFiniteAndPositive( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set, finite, // and non-negative. Otherwise, will return an `error_code` error status that // includes `name` in the error message. absl::Status ValidateIsFiniteAndNonNegative( - absl::optional opt, absl::string_view name, + std::optional opt, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and @@ -466,7 +466,7 @@ absl::Status ValidateIsFiniteAndNonNegative( // Otherwise, will return an `error_code` error status that includes `name` in // the error message. absl::Status ValidateIsInInclusiveInterval( - absl::optional opt, double lower_bound, double upper_bound, + std::optional opt, double lower_bound, double upper_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); @@ -475,7 +475,7 @@ absl::Status ValidateIsInInclusiveInterval( // Otherwise, will return an `error_code` error status that includes `name` in // the error message. absl::Status ValidateIsInExclusiveInterval( - absl::optional opt, double lower_bound, double upper_bound, + std::optional opt, double lower_bound, double upper_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); @@ -483,28 +483,28 @@ absl::Status ValidateIsInExclusiveInterval( // strictly lesser than `upper_bound`. Otherwise, will return an `error_code` // error status that includes `name` in the error message. absl::Status ValidateIsLesserThan( - absl::optional opt, double upper_bound, absl::string_view name, + std::optional opt, double upper_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // lesser than or equal to `upper_bound`. Otherwise, will return an `error_code` // error status that includes `name` in the error message. absl::Status ValidateIsLesserThanOrEqualTo( - absl::optional opt, double upper_bound, absl::string_view name, + std::optional opt, double upper_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // strictly greater than `lower_bound`. Otherwise, will return an `error_code` // error status that includes `name` in the error message. absl::Status ValidateIsGreaterThan( - absl::optional opt, double lower_bound, absl::string_view name, + std::optional opt, double lower_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and // greater than or equal to `upper_bound`. Otherwise, will return an // `error_code` error status that includes `name` in the error message. absl::Status ValidateIsGreaterThanOrEqualTo( - absl::optional opt, double lower_bound, absl::string_view name, + std::optional opt, double lower_bound, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Returns absl::OkStatus() if the value of optional `opt` if it is set and @@ -513,25 +513,25 @@ absl::Status ValidateIsGreaterThanOrEqualTo( // true, respectively. Otherwise, will return an `error_code` error status that // includes `name` in the error message. absl::Status ValidateIsInInterval( - absl::optional opt, double lower_bound, double upper_bound, + std::optional opt, double lower_bound, double upper_bound, bool include_lower, bool include_upper, absl::string_view name, absl::StatusCode error_code = absl::StatusCode::kInvalidArgument); // Methods for semantical and consistent validation of common parameters. -absl::Status ValidateEpsilon(absl::optional epsilon); -absl::Status ValidateDelta(absl::optional delta); +absl::Status ValidateEpsilon(std::optional epsilon); +absl::Status ValidateDelta(std::optional delta); absl::Status ValidateMaxPartitionsContributed( - absl::optional max_partitions_contributed); + std::optional max_partitions_contributed); absl::Status ValidateMaxContributionsPerPartition( - absl::optional max_contributions_per_partition); -absl::Status ValidateMaxContributions(absl::optional max_contributions); + std::optional max_contributions_per_partition); +absl::Status ValidateMaxContributions(std::optional max_contributions); // Validates common tree parameters. -absl::Status ValidateTreeHeight(absl::optional tree_height); -absl::Status ValidateBranchingFactor(absl::optional branching_factor); +absl::Status ValidateTreeHeight(std::optional tree_height); +absl::Status ValidateBranchingFactor(std::optional branching_factor); template -absl::Status ValidateBounds(absl::optional lower, absl::optional upper) { +absl::Status ValidateBounds(std::optional lower, std::optional upper) { if (!lower.has_value() && !upper.has_value()) { return absl::OkStatus(); } @@ -552,12 +552,12 @@ absl::Status ValidateBounds(absl::optional lower, absl::optional upper) { return absl::OkStatus(); } -absl::Status ValidatePreThresholdOptional(absl::optional pre_threshold); +absl::Status ValidatePreThresholdOptional(std::optional pre_threshold); [[deprecated( "This validator is used for a class that is deprecated in favour of the " "pre_threshold attribute of other strategies classes.")]] absl::Status -ValidatePreThreshold(absl::optional pre_threshold); +ValidatePreThreshold(std::optional pre_threshold); } // namespace differential_privacy #endif // DIFFERENTIAL_PRIVACY_ALGORITHMS_UTIL_H_ diff --git a/cc/algorithms/util_test.cc b/cc/algorithms/util_test.cc index b53181e0..1a9c38ea 100644 --- a/cc/algorithms/util_test.cc +++ b/cc/algorithms/util_test.cc @@ -489,7 +489,7 @@ TEST(SafeCastFromDoubleTest, ForFloat) { } TEST(ValidateTest, IsSet) { - absl::optional opt; + std::optional opt; EXPECT_THAT(ValidateIsSet(opt, "Test value"), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Test value must be set."))); @@ -1161,11 +1161,11 @@ TEST(ValidateTest, ValidateBoundsChecksOrder) { } TEST(ValidateTest, ValidateBoundsChecksBothSetOrUnset) { - EXPECT_THAT(ValidateBounds(1, absl::nullopt), + EXPECT_THAT(ValidateBounds(1, std::nullopt), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("both"))); - EXPECT_THAT(ValidateBounds(absl::nullopt, 1), + EXPECT_THAT(ValidateBounds(std::nullopt, 1), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("both"))); - EXPECT_THAT(ValidateBounds(absl::nullopt, absl::nullopt), + EXPECT_THAT(ValidateBounds(std::nullopt, std::nullopt), StatusIs(absl::StatusCode::kOk)); } @@ -1188,7 +1188,7 @@ TEST(ValidateTest, ValidateTreeHeightForInvalidNumeric) { } TEST(ValidateTest, ValidateTreeHeightForEmpty) { - EXPECT_THAT(ValidateTreeHeight(absl::nullopt), + EXPECT_THAT(ValidateTreeHeight(std::nullopt), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Tree Height must be set."))); } @@ -1216,7 +1216,7 @@ TEST(ValidateTest, ValidateBranchingFactorForInvalidNumeric) { } TEST(ValidateTest, ValidateBranchingFactorForEmpty) { - EXPECT_THAT(ValidateBranchingFactor(absl::nullopt), + EXPECT_THAT(ValidateBranchingFactor(std::nullopt), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Branching Factor must be set"))); } diff --git a/cc/cc_differential_privacy_deps.bzl b/cc/cc_differential_privacy_deps.bzl index 298ad95c..34dda2c0 100644 --- a/cc/cc_differential_privacy_deps.bzl +++ b/cc/cc_differential_privacy_deps.bzl @@ -26,18 +26,18 @@ def cc_differential_privacy_deps(): # Abseil http_archive( name = "com_google_absl", - url = "https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.2.tar.gz", - sha256 = "733726b8c3a6d39a4120d7e45ea8b41a434cdacde401cba500f14236c49b39dc", - strip_prefix = "abseil-cpp-20240116.2", + url = "https://github.com/abseil/abseil-cpp/archive/refs/tags/20240722.0.tar.gz", + sha256 = "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3", + strip_prefix = "abseil-cpp-20240722.0", ) # Common bazel rules. Also required for Abseil. http_archive( name = "bazel_skylib", - sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", + sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.7.1/bazel-skylib-1.7.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.1/bazel-skylib-1.7.1.tar.gz", ], ) diff --git a/examples/java/BUILD b/examples/java/BUILD index 405a8275..05d9e508 100644 --- a/examples/java/BUILD +++ b/examples/java/BUILD @@ -12,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# + +load("@rules_java//java:defs.bzl", "java_binary") package( default_visibility = [ diff --git a/java/main/com/google/privacy/differentialprivacy/BUILD b/java/main/com/google/privacy/differentialprivacy/BUILD index 451e7d81..7e7af902 100644 --- a/java/main/com/google/privacy/differentialprivacy/BUILD +++ b/java/main/com/google/privacy/differentialprivacy/BUILD @@ -12,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# + +load("@rules_java//java:defs.bzl", "java_library") package( default_visibility = [ diff --git a/java/main/com/google/privacy/differentialprivacy/testing/BUILD b/java/main/com/google/privacy/differentialprivacy/testing/BUILD index 9072513e..451fee01 100644 --- a/java/main/com/google/privacy/differentialprivacy/testing/BUILD +++ b/java/main/com/google/privacy/differentialprivacy/testing/BUILD @@ -12,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# + +load("@rules_java//java:defs.bzl", "java_library") package( default_visibility = [ diff --git a/java/tests/com/google/privacy/differentialprivacy/BUILD b/java/tests/com/google/privacy/differentialprivacy/BUILD index 806e32db..0c73b78f 100644 --- a/java/tests/com/google/privacy/differentialprivacy/BUILD +++ b/java/tests/com/google/privacy/differentialprivacy/BUILD @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package( default_visibility = [ "//visibility:public", diff --git a/java/tests/com/google/privacy/differentialprivacy/statistical/BUILD b/java/tests/com/google/privacy/differentialprivacy/statistical/BUILD index 1f7c5260..d764dfd8 100644 --- a/java/tests/com/google/privacy/differentialprivacy/statistical/BUILD +++ b/java/tests/com/google/privacy/differentialprivacy/statistical/BUILD @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library", "java_test") + package( default_visibility = [ "//visibility:public", diff --git a/java/tests/com/google/privacy/differentialprivacy/testing/BUILD b/java/tests/com/google/privacy/differentialprivacy/testing/BUILD index 1811af38..3b23053f 100644 --- a/java/tests/com/google/privacy/differentialprivacy/testing/BUILD +++ b/java/tests/com/google/privacy/differentialprivacy/testing/BUILD @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library", "java_test") + package( default_visibility = [ "//visibility:public", diff --git a/privacy-on-beam/pbeam/mean_test.go b/privacy-on-beam/pbeam/mean_test.go index a7dc0e07..632823af 100644 --- a/privacy-on-beam/pbeam/mean_test.go +++ b/privacy-on-beam/pbeam/mean_test.go @@ -819,9 +819,18 @@ func TestMeanPerKeyCountsPrivacyUnitIDsWithMultipleContributionsCorrectly(t *tes // because the probability of keeping the partition with 2 elements is negligible, ≈5.184e-179. testutils.MakeTripleWithFloatValueStartingFromKey(18, 2, 2, 0)) + // Duplicated contribution to partition 2 from privacy unit 18 and 19, each duplicated 20 times. + // While these duplicates are kept for computing the mean (due to maxContributionsPerPartition=20), + // they are not considered for partition selection. + // There are only two privacy units in partition 2, which is smaller than the threshold and should be dropped. for i := 0; i < 20; i++ { triples = append(triples, testutils.MakeTripleWithFloatValueStartingFromKey(18, 2, 2, 1)...) } + + // The threshold is equal to 11. + // The probability to keep partition 0 (count = 7) is 1.942e-70. + // The probability to keep partition 2 (count = 2) is 5.184e-179. + // So only partition 1 is kept. exactCount := 11.0 exactMean := 1.3 result := []testutils.PairIF64{ diff --git a/proto/data.proto b/proto/data.proto index 7467123c..2e0dfdbc 100644 --- a/proto/data.proto +++ b/proto/data.proto @@ -1,4 +1,3 @@ -// // Copyright 2019 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// // This file defines the input/output data format accepted by the differentially // private algorithms. @@ -40,11 +38,13 @@ message Output { message Element { // Diff. priv. result of the operation performed over the input data. optional ValueType value = 1; + // Confidence interval of the noise added to the result optional ConfidenceInterval noise_confidence_interval = 3; reserved 2; } + repeated Element elements = 1; // Contains information about algorithm accuracy. @@ -53,6 +53,7 @@ message Output { reserved 1; } + // Error report is attached if either the noise confidence interval or the // bounding report is available. optional ErrorReport error_report = 3; diff --git a/python/dp_auditorium/dp_auditorium/configs/dataset_generator_config.py b/python/dp_auditorium/dp_auditorium/configs/dataset_generator_config.py index 677e3eb4..0f38d689 100644 --- a/python/dp_auditorium/dp_auditorium/configs/dataset_generator_config.py +++ b/python/dp_auditorium/dp_auditorium/configs/dataset_generator_config.py @@ -18,7 +18,6 @@ from vizier.service import pyvizier as vz - class DataType(enum.Enum): DATA_TYPE_UNSPECIFIED = 0 DATA_TYPE_INT32 = 1 @@ -52,7 +51,7 @@ class VizierDatasetGeneratorConfig: data_type: DataType min_value: float max_value: float - search_algorithm: vz.Algorithm + search_algorithm: str metric_name: str @@ -83,6 +82,7 @@ class ClassificationDatasetGeneratorConfig: num_samples: Number of samples in the largest dataset. num_classes: Number of classes in the classification task. """ + study_name: str study_owner: str min_value: float