Add the DP-Auditorium Python library

- Initial release of DP-Auditorium (experimental).
- Supports the main testing interfaces and some simple test mechanisms.
- Includes a demo with a mean mechanism.

PiperOrigin-RevId: 605711565
Change-Id: Ic8da18530eaa3317aff6eb147302cd1bd4122132
GitOrigin-RevId: 8a5d2830fcaa1a85d6e05f972892632a15cb61b9

java/tests/com/google/privacy/differentialprivacy/ApproximateBoundsTest.java

@@ -25,8 +25,8 @@
 import com.google.privacy.differentialprivacy.proto.SummaryOuterClass.ApproxBoundsSummary;
 import com.google.protobuf.ExtensionRegistry;
 import com.google.testing.junit.testparameterinjector.TestParameter;
-import com.google.testing.junit.testparameterinjector.TestParameter.TestParameterValuesProvider;
 import com.google.testing.junit.testparameterinjector.TestParameterInjector;
+import com.google.testing.junit.testparameterinjector.TestParameterValuesProvider;
 import java.util.List;
 import org.junit.Before;
 import org.junit.Test;
@@ -406,16 +406,16 @@ public void mergeWith_calledAfterSerialization_throws() {
     assertThat(thrown).hasMessageThat().contains("already serialized");
   }
 
-  private static class SuccessProbabilityProvider implements TestParameterValuesProvider {
+  private static class SuccessProbabilityProvider extends TestParameterValuesProvider {
     @Override
-    public List<Double> provideValues() {
+    public List<Double> provideValues(Context context) {
       return ImmutableList.of(1e-11, 1e-5, 0.1, 0.25, 0.5, 0.75, 0.9, 1 - 1e-5, 1 - 1e-15);
     }
   }
 
-  private static class NumBinsProvider implements TestParameterValuesProvider {
+  private static class NumBinsProvider extends TestParameterValuesProvider {
     @Override
-    public List<Integer> provideValues() {
+    public List<Integer> provideValues(Context context) {
       return ImmutableList.of(1, 2, 4, 1024, 2048);
     }
   }

python/dp_auditorium/.bazelversion

@@ -0,0 +1 @@
+6.4.0

python/dp_auditorium/BUILD.bazel

@@ -0,0 +1,15 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/dp_auditorium/README.md

@@ -0,0 +1,64 @@
+### DP-Auditorium
+
+This folder contains the DP-Auditorium library for auditing differential privacy
+guarantees as described in the introductory paper
+[DP-Auditorium: a Large Scale Library for Auditing Differential Privacy](https://arxiv.org/abs/2307.05608).
+
+The root directory is structured as follows. It contains four main folders and a
+runner module described below:
+
+*   [Testers](testers/): Contains modules introducing `PropertyTesters` that
+    check if there is evidence to reject the hypothesis that a privacy guarantee
+    holds on a fixed pair of datasets.
+*   [Generators](generators/): Contains modules introducing `DatasetGenerators`
+    that output pairs of neighboring datasets, for example under the add/remove
+    neighboring relation.
+*   [Mechanisms](mechanims/): Contains examples of private and non-private
+    mechanisms used to exemplify testers.
+*   `privacy_test_runner.py`: Module with runner that specializes to a tester
+    and generator and instantiates the `PropertyTester` on several datasets'
+    trials generated by the `DatasetGenerator`.
+*   [Examples](examples/): Folder containing examples on how to combine the
+    above tools.
+
+Details on the signature of the above objects, can be found in the
+`interfaces.py` module. For further details please refer to the paper
+[DP-Auditorium: a Large Scale Library for Auditing Differential Privacy.](https://arxiv.org/abs/2307.05608)
+
+## Examples
+
+An illustrative example is available in
+`examples/run_mean_mechanism_example.py`. This binary defines a Hockey-Stick
+divergence test as the property tester and a dataset generator that employs a
+Gaussian process bandit (using OSS Vizier) for suggesting datasets.
+Subsequently, the runner is instantiated with these two objects to conduct a
+test on a non-private mean mechanism.
+
+There are two ways to run this, either via Bazel or after installing the library
+using `setup.py`.
+
+### Run with bazel
+
+For the first option, you need to have
+[Bazel installed](https://docs.bazel.build/versions/main/install.html).
+Once that is done, run:
+
+```
+bazel build dp_auditorium:all
+bazel run dp_auditorium/examples:run_mean_mechanism_example
+```
+
+### Run via setup.py
+
+For the second option, you will need the
+[setuptools package](https://pypi.org/project/setuptools/) installed.
+To ensure this, you may run
+```
+pip install --upgrade setuptools
+```
+Then, to demonstrate our example, run:
+```
+python setup.py install
+python dp_auditorium/examples/run_mean_mechanism_example.py
+```
+

python/dp_auditorium/VERSION

@@ -0,0 +1,2 @@
+""" Version of the current release of the DP testing library."""
+0.0.1

python/dp_auditorium/WORKSPACE

@@ -0,0 +1,39 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+workspace(name = "dp_auditorium_py")
+
+load("@dp_auditorium_py//:dp_auditorium_py_deps.bzl", "dp_auditorium_py_deps")
+
+dp_auditorium_py_deps()
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+load("@dp_auditorium_py//:dp_auditorium_py_deps_init.bzl", "dp_auditorium_py_deps_init")
+
+dp_auditorium_py_deps_init("dp_auditorium_py")
+
+load("@dp_auditorium_py_pip_deps//:requirements.bzl", "install_deps")
+
+install_deps()

python/dp_auditorium/dp_auditorium/BUILD.bazel

@@ -0,0 +1,66 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Top-level APIs
+
+load("@dp_auditorium_py_pip_deps//:requirements.bzl", "requirement")
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "dp_auditorium",
+    srcs = ["__init__.py"],
+    deps = [
+        ":interfaces",
+        ":privacy_test_runner",
+        "//dp_auditorium/generators",
+        "//dp_auditorium/mechanisms",
+        "//dp_auditorium/testers",
+    ],
+)
+
+py_library(
+    name = "interfaces",
+    srcs = ["interfaces.py"],
+    deps = [
+        "//dp_auditorium/configs",
+        requirement("numpy"),
+    ],
+)
+
+py_library(
+    name = "privacy_test_runner",
+    srcs = ["privacy_test_runner.py"],
+    deps = [
+        ":interfaces",
+        "//dp_auditorium/configs",
+        requirement("numpy"),
+        requirement("scipy"),
+    ],
+)
+
+py_test(
+    name = "privacy_test_runner_test",
+    srcs = ["privacy_test_runner_test.py"],
+    deps = [
+        ":interfaces",
+        ":privacy_test_runner",
+        "//dp_auditorium/configs",
+        requirement("absl-py"),
+        requirement("numpy"),
+        requirement("tensorflow"),
+    ],
+)

python/dp_auditorium/dp_auditorium/__init__.py

@@ -0,0 +1,22 @@
+# Copyright 2024 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Differential Privacy Testing Library."""
+
+from dp_auditorium import configs
+from dp_auditorium import generators
+from dp_auditorium import interfaces
+from dp_auditorium import mechanisms
+from dp_auditorium import privacy_test_runner
+from dp_auditorium import testers

python/dp_auditorium/dp_auditorium/configs/BUILD.bazel

@@ -0,0 +1,33 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Differential Privacy Data Generators.
+
+load("@dp_auditorium_py_pip_deps//:requirements.bzl", "requirement")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+py_library(
+    name = "configs",
+    srcs = [
+        "dataset_generator_config.py",
+        "mechanism_config.py",
+        "privacy_property.py",
+        "privacy_test_runner_config.py",
+        "property_tester_config.py",
+    ],
+)

python/dp_auditorium/dp_auditorium/configs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

python/dp_auditorium/dp_auditorium/configs/dataset_generator_config.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration for Vizier dataset generators."""
+
+import dataclasses
+import enum
+
+
+class DataType(enum.Enum):
+  DATA_TYPE_UNSPECIFIED = 0
+  DATA_TYPE_INT32 = 1
+  DATA_TYPE_FLOAT = 2
+
+
+@dataclasses.dataclass
+class VizierDatasetGeneratorConfig:
+  """Configuration for Vizier dataset generators.
+
+  Attributes:
+    study_name: String passed to Vizier to identify the study.
+    study_owner: String determining the owner of the study. A Vizier client
+      organizes studies by `owner` and `study_id`. For details see the class
+      `Study` in `vizier/_src/service/clients.py`
+      (https://github.com/google/vizier/tree/main).
+    num_vizier_parameters: Number of parameters created by vizier.
+    data_type: Data type of the Vizier parameters.
+    min_value: Minimum value of the Vizier parameters.
+    max_value: Maximum value of the Vizier parameters.
+    search_algorithm: Search algorithm used by Vizier to generate parameters.
+      See `vizier/_src/pyvizier/oss/study_config.py` for possible values.
+    metric_name: User-specifed key, which is passed to Vizier for the purpose of
+      storing metric values to be optimized. For instance, in a test using the
+      `RenyiPropertyTester`, this can be denoted as `renyi_divergence`.
+  """
+
+  study_name: str
+  study_owner: str
+  num_vizier_parameters: int
+  data_type: DataType
+  min_value: float
+  max_value: float
+  search_algorithm: str
+  metric_name: str