From fb99350b55cae06d73dc1c3effa410ea0334f532 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 4 Mar 2020 13:22:09 -0600 Subject: [PATCH] WIP: asynchromix docs * Add documentation for asynchromix app. * Remove extra whitespace in asynchromix contract * Add contract under contracts/ - that is the default location for documentation purposes when using sphinxcontrib-soliditydomain. --- apps/asynchromix/asynchromix.py | 70 ++++- apps/asynchromix/asynchromix.sol | 75 ++--- apps/asynchromix/powermixing.py | 10 + contracts/asynchromix.sol | 254 +++++++++++++++++ docker-compose.yml | 4 + docs/conf.py | 5 +- docs/index.rst | 3 +- docs/integrations/eth.rst | 461 +++++++++++++++++++++++++++++++ docs/integrations/refs.bib | 16 ++ honeybadgermpc/preprocessing.py | 35 ++- setup.py | 2 + 11 files changed, 877 insertions(+), 58 deletions(-) create mode 100644 contracts/asynchromix.sol create mode 100644 docs/integrations/eth.rst create mode 100644 docs/integrations/refs.bib diff --git a/apps/asynchromix/asynchromix.py b/apps/asynchromix/asynchromix.py index 547e3cd0..7e18e9fd 100644 --- a/apps/asynchromix/asynchromix.py +++ b/apps/asynchromix/asynchromix.py @@ -52,8 +52,30 @@ async def wait_for_receipt(w3, tx_hash): ######## -class AsynchromixClient(object): +class AsynchromixClient: + """An Asynchromix client sends "masked" messages to an Ethereum contract. + ... + """ + def __init__(self, sid, myid, send, recv, w3, contract, req_mask): + """ + Parameters + ---------- + sid: int + Session id. + myid: int + Client id. + send: + Function used to send messages. Not used? + recv: + Function used to receive messages. Not used? + w3: + Connection instance to an Ethereum node. + contract: + Contract instance on the Ethereum blockchain. + req_mask: + Function used to request an input mask from a server. + """ self.sid = sid self.myid = myid self.contract = contract @@ -66,7 +88,8 @@ async def _run(self): contract_concise = ConciseContract(self.contract) await asyncio.sleep(60) # give the servers a head start # Client sends several batches of messages then quits - for epoch in range(1000): + # for epoch in range(1000): + for epoch in range(10): logging.info(f"[Client] Starting Epoch {epoch}") receipts = [] for i in range(32): @@ -125,13 +148,13 @@ async def send_message(self, m): # Step 3. Fetch the input mask from the servers inputmask = await self._get_inputmask(inputmask_idx) message = int.from_bytes(m.encode(), "big") - maskedinput = message + inputmask - maskedinput_bytes = self.w3.toBytes(hexstr=hex(maskedinput.value)) - maskedinput_bytes = maskedinput_bytes.rjust(32, b"\x00") + masked_message = message + inputmask + masked_message_bytes = self.w3.toBytes(hexstr=hex(masked_message.value)) + masked_message_bytes = masked_message_bytes.rjust(32, b"\x00") # Step 4. Publish the masked input tx_hash = self.contract.functions.submit_message( - inputmask_idx, maskedinput_bytes + inputmask_idx, masked_message_bytes ).transact({"from": self.w3.eth.accounts[0]}) tx_receipt = await wait_for_receipt(self.w3, tx_hash) @@ -142,19 +165,42 @@ async def send_message(self, m): class AsynchromixServer(object): + """Asynchromix server class to ...""" + def __init__(self, sid, myid, send, recv, w3, contract): + """ + Parameters + ---------- + sid: int + Session id. + myid: int + Client id. + send: + Function used to send messages. + recv: + Function used to receive messages. + w3: + Connection instance to an Ethereum node. + contract: + Contract instance on the Ethereum blockchain. + """ self.sid = sid self.myid = myid self.contract = contract self.w3 = w3 + self._task1a = asyncio.ensure_future(self._offline_inputmasks_loop()) self._task1a.add_done_callback(print_exception_callback) + self._task1b = asyncio.ensure_future(self._offline_mixes_loop()) self._task1b.add_done_callback(print_exception_callback) + self._task2 = asyncio.ensure_future(self._client_request_loop()) self._task2.add_done_callback(print_exception_callback) + self._task3 = asyncio.ensure_future(self._mixing_loop()) self._task3.add_done_callback(print_exception_callback) + self._task4 = asyncio.ensure_future(self._mixing_initiate_loop()) self._task4.add_done_callback(print_exception_callback) @@ -186,7 +232,7 @@ async def join(self): The bits and triples are consumed by each mixing epoch. The input masks may be claimed at a different rate than - than the mixing epochs so they are replenished in a separate + the mixing epochs so they are replenished in a separate task """ @@ -326,13 +372,13 @@ async def _mixing_loop(self): # 3.b. Collect the inputs inputs = [] for idx in range(epoch * K, (epoch + 1) * K): - # Get the public input - masked_input, inputmask_idx = contract_concise.input_queue(idx) - masked_input = field(int.from_bytes(masked_input, "big")) - # Get the input masks + # Get the public input (masked message) + masked_message_bytes, inputmask_idx = contract_concise.input_queue(idx) + masked_message = field(int.from_bytes(masked_message_bytes, "big")) + # Get the input mask inputmask = self._inputmasks[inputmask_idx] - m_share = masked_input - inputmask + m_share = masked_message - inputmask inputs.append(m_share) # 3.c. Collect the preprocessing diff --git a/apps/asynchromix/asynchromix.sol b/apps/asynchromix/asynchromix.sol index a0f16672..cfa279e8 100644 --- a/apps/asynchromix/asynchromix.sol +++ b/apps/asynchromix/asynchromix.sol @@ -8,13 +8,13 @@ contract AsynchromixCoordinator { * 3. Initiates mixing epochs (MPC computations) * (makes use of preprocess triples, bits, powers) */ - + // Session parameters uint public n; uint public t; address[] public servers; mapping (address => uint) public servermap; - + constructor(address[] _servers, uint _t) public { n = _servers.length; t = _t; @@ -34,7 +34,7 @@ contract AsynchromixCoordinator { "0xca35b7d915458ef540ade6068dfe2f44e8fa733c"] * */ - + // ############################################### // 1. Preprocessing Buffer (the MPC offline phase) // ############################################### @@ -44,26 +44,26 @@ contract AsynchromixCoordinator { uint bits; // [b] with b in {-1,1} uint inputmasks; // [r] } - + // Consensus count (min of the player report counts) PreProcessCount public preprocess; - + // How many of each have been reserved already PreProcessCount public preprocess_used; function inputmasks_available () public view returns(uint) { return preprocess.inputmasks - preprocess_used.inputmasks; - } + } // Report of preprocess buffer size from each server mapping ( uint => PreProcessCount ) public preprocess_reports; - + event PreProcessUpdated(); - + function min(uint a, uint b) private pure returns (uint) { return a < b ? a : b; - } - + } + function max(uint a, uint b) private pure returns (uint) { return a > b ? a : b; } @@ -96,27 +96,27 @@ contract AsynchromixCoordinator { preprocess.bits = mins.bits; preprocess.inputmasks = mins.inputmasks; } - - - + + + // ###################### // 2. Accept client input // ###################### - + // Step 2.a. Clients can reserve an input mask [r] from Preprocessing // maps each element of preprocess.inputmasks to the client (if any) that claims it mapping (uint => address) public inputmasks_claimed; - + event InputMaskClaimed(address client, uint inputmask_idx); - + // Client reserves a random values function reserve_inputmask() public returns(uint) { // Extension point: override this function to add custom token rules - + // An unclaimed input mask must already be available require(preprocess.inputmasks > preprocess_used.inputmasks); - + // Acquire this input mask for msg.sender uint idx = preprocess_used.inputmasks; inputmasks_claimed[idx] = msg.sender; @@ -124,57 +124,58 @@ contract AsynchromixCoordinator { emit InputMaskClaimed(msg.sender, idx); return idx; } - + // Step 2.b. Client requests (out of band, e.g. over https) shares of [r] // from each server. Servers use this function to check authorization. // Authentication using client's address is also out of band function client_authorized(address client, uint idx) view public returns(bool) { return inputmasks_claimed[idx] == client; } - + // Step 2.c. Clients publish masked message (m+r) to provide a new input [m] // and bind it to the preprocess input mapping (uint => bool) public inputmask_map; // Maps a mask - + struct Input { bytes32 masked_input; // (m+r) uint inputmask; // index in inputmask of mask [r] // Extension point: add more metadata about each input } - + Input[] public input_queue; // All inputs sent so far function input_queue_length() public view returns(uint) { return input_queue.length; } - + event MessageSubmitted(uint idx, uint inputmask_idx, bytes32 masked_input); function submit_message(uint inputmask_idx, bytes32 masked_input) public { // Must be authorized to use this input mask require(inputmasks_claimed[inputmask_idx] == msg.sender); - + // Extension point: add additional client authorizations, // e.g. prevent the client from submitting more than one message per mix - + uint idx = input_queue.length; input_queue.length += 1; - + input_queue[idx].masked_input = masked_input; input_queue[idx].inputmask = inputmask_idx; - + + // QUESTION: What is the purpose of this event? emit MessageSubmitted(idx, inputmask_idx, masked_input); // The input masks are deactivated after first use inputmasks_claimed[inputmask_idx] = address(0); } - + // ######################### // 3. Initiate Mixing Epochs // ######################### - + uint public constant K = 32; // Mix Size - + // Preprocessing requirements uint public constant PER_MIX_TRIPLES = (K / 2) * 5 * 5; // k log^2 k uint public constant PER_MIX_BITS = (K / 2) * 5 * 5; @@ -187,7 +188,7 @@ contract AsynchromixCoordinator { return min(triples_available / PER_MIX_TRIPLES, bits_available / PER_MIX_BITS); } - + // Step 3.a. Trigger a mix to start uint public inputs_mixed; uint public epochs_initiated; @@ -196,27 +197,27 @@ contract AsynchromixCoordinator { function inputs_ready() public view returns(uint) { return input_queue.length - inputs_mixed; } - + function initiate_mix() public { // Must mix eactly K values in each epoch require(input_queue.length >= inputs_mixed + K); - + // Can only initiate mix if enough preprocessings are ready require(preprocess.triples >= preprocess_used.triples + PER_MIX_TRIPLES); require(preprocess.bits >= preprocess_used.bits + PER_MIX_BITS); preprocess_used.triples += PER_MIX_TRIPLES; preprocess_used.bits += PER_MIX_BITS; - + inputs_mixed += K; emit MixingEpochInitiated(epochs_initiated); epochs_initiated += 1; output_votes.length = epochs_initiated; output_hashes.length = epochs_initiated; } - + // Step 3.b. Output reporting: the output is considered "approved" once // at least t+1 servers report it - + uint public outputs_ready; event MixOutput(uint epoch, string output); bytes32[] public output_hashes; @@ -242,7 +243,7 @@ contract AsynchromixCoordinator { } else { output_hashes[epoch] = output_hash; } - + output_votes[epoch] += 1; if (output_votes[epoch] == t + 1) { // at least one honest node agrees emit MixOutput(epoch, output); diff --git a/apps/asynchromix/powermixing.py b/apps/asynchromix/powermixing.py index 29d91e6f..f6df5b45 100644 --- a/apps/asynchromix/powermixing.py +++ b/apps/asynchromix/powermixing.py @@ -168,6 +168,7 @@ async def async_mixing_in_processes(network_info, n, t, k, run_id, node_id): if __name__ == "__main__": from honeybadgermpc.config import HbmpcConfig + logging.info("Running powermixing app ...") HbmpcConfig.load_config() run_id = HbmpcConfig.extras["run_id"] @@ -181,6 +182,10 @@ async def async_mixing_in_processes(network_info, n, t, k, run_id, node_id): try: if not HbmpcConfig.skip_preprocessing: + logging.info( + "Running preprocessing.\n" + 'To skip preprocessing phase set "skip_preprocessing" config to true.' + ) # Need to keep these fixed when running on processes. field = GF(Subgroup.BLS12_381) a_s = [field(i) for i in range(1000 + k, 1000, -1)] @@ -191,6 +196,11 @@ async def async_mixing_in_processes(network_info, n, t, k, run_id, node_id): pp_elements.preprocessing_done() else: loop.run_until_complete(pp_elements.wait_for_preprocessing()) + else: + logging.info( + "Skipping preprocessing.\n" + 'To run preprocessing phase set "skip_preprocessing" config to false.' + ) loop.run_until_complete( async_mixing_in_processes( diff --git a/contracts/asynchromix.sol b/contracts/asynchromix.sol new file mode 100644 index 00000000..ebe462e7 --- /dev/null +++ b/contracts/asynchromix.sol @@ -0,0 +1,254 @@ +pragma solidity >=0.4.22 <0.6.0; + +/// @title A blockchain-based MPC coordinator for Asychromix. +/// @author Andrew Miller +contract AsynchromixCoordinator { + /* A blockchain-based MPC coordinator for Asychromix. + * 1. Keeps track of the MPC "preprocessing buffer" + * 2. Accepts client input + * (makes use of preprocess randoms) + * 3. Initiates mixing epochs (MPC computations) + * (makes use of preprocess triples, bits, powers) + */ + + // Session parameters + uint public n; + uint public t; + address[] public servers; + mapping (address => uint) public servermap; + + constructor(address[] _servers, uint _t) public { + n = _servers.length; + t = _t; + require(3*t < n); + servers.length = n; + for (uint i = 0; i < n; i++) { + servers[i] = _servers[i]; + servermap[_servers[i]] = i+1; // servermap is off-by-one + } + } + /* + * It's necessary to paste JSON into the "_servers" constructor to use the Remix IDE + * Copy and paste the following for n=4: +["0xca35b7d915458ef540ade6068dfe2f44e8fa733c", + "0xca35b7d915458ef540ade6068dfe2f44e8fa733c", + "0xca35b7d915458ef540ade6068dfe2f44e8fa733c", + "0xca35b7d915458ef540ade6068dfe2f44e8fa733c"] + * + */ + + // ############################################### + // 1. Preprocessing Buffer (the MPC offline phase) + // ############################################### + + struct PreProcessCount { + uint triples; // [a],[b],[ab] + uint bits; // [b] with b in {-1,1} + uint inputmasks; // [r] + } + + // Consensus count (min of the player report counts) + PreProcessCount public preprocess; + + // How many of each have been reserved already + PreProcessCount public preprocess_used; + + function inputmasks_available () public view returns(uint) { + return preprocess.inputmasks - preprocess_used.inputmasks; + } + + // Report of preprocess buffer size from each server + mapping ( uint => PreProcessCount ) public preprocess_reports; + + event PreProcessUpdated(); + + function min(uint a, uint b) private pure returns (uint) { + return a < b ? a : b; + } + + function max(uint a, uint b) private pure returns (uint) { + return a > b ? a : b; + } + + function preprocess_report(uint[3] rep) public { + // Update the Report + require(servermap[msg.sender] > 0); // only valid servers + uint id = servermap[msg.sender] - 1; + preprocess_reports[id].triples = rep[0]; + preprocess_reports[id].bits = rep[1]; + preprocess_reports[id].inputmasks = rep[2]; + + // Update the consensus + // .triples = min (over each id) of _reports[id].triples; same for bits, etc. + PreProcessCount memory mins; + mins.triples = preprocess_reports[0].triples; + mins.bits = preprocess_reports[0].bits; + mins.inputmasks = preprocess_reports[0].inputmasks; + for (uint i = 1; i < n; i++) { + mins.triples = min(mins.triples, preprocess_reports[i].triples); + mins.bits = min(mins.bits, preprocess_reports[i].bits); + mins.inputmasks = min(mins.inputmasks, preprocess_reports[i].inputmasks); + } + if (preprocess.triples < mins.triples || + preprocess.bits < mins.bits || + preprocess.inputmasks < mins.inputmasks) { + emit PreProcessUpdated(); + } + preprocess.triples = mins.triples; + preprocess.bits = mins.bits; + preprocess.inputmasks = mins.inputmasks; + } + + + + // ###################### + // 2. Accept client input + // ###################### + + // Step 2.a. Clients can reserve an input mask [r] from Preprocessing + + // maps each element of preprocess.inputmasks to the client (if any) that claims it + mapping (uint => address) public inputmasks_claimed; + + event InputMaskClaimed(address client, uint inputmask_idx); + + // Client reserves a random values + function reserve_inputmask() public returns(uint) { + // Extension point: override this function to add custom token rules + + // An unclaimed input mask must already be available + require(preprocess.inputmasks > preprocess_used.inputmasks); + + // Acquire this input mask for msg.sender + uint idx = preprocess_used.inputmasks; + inputmasks_claimed[idx] = msg.sender; + preprocess_used.inputmasks += 1; + emit InputMaskClaimed(msg.sender, idx); + return idx; + } + + // Step 2.b. Client requests (out of band, e.g. over https) shares of [r] + // from each server. Servers use this function to check authorization. + // Authentication using client's address is also out of band + function client_authorized(address client, uint idx) view public returns(bool) { + return inputmasks_claimed[idx] == client; + } + + // Step 2.c. Clients publish masked message (m+r) to provide a new input [m] + // and bind it to the preprocess input + mapping (uint => bool) public inputmask_map; // Maps a mask + + struct Input { + bytes32 masked_input; // (m+r) + uint inputmask; // index in inputmask of mask [r] + + // Extension point: add more metadata about each input + } + + Input[] public input_queue; // All inputs sent so far + function input_queue_length() public view returns(uint) { + return input_queue.length; + } + + event MessageSubmitted(uint idx, uint inputmask_idx, bytes32 masked_input); + + function submit_message(uint inputmask_idx, bytes32 masked_input) public { + // Must be authorized to use this input mask + require(inputmasks_claimed[inputmask_idx] == msg.sender); + + // Extension point: add additional client authorizations, + // e.g. prevent the client from submitting more than one message per mix + + uint idx = input_queue.length; + input_queue.length += 1; + + input_queue[idx].masked_input = masked_input; + input_queue[idx].inputmask = inputmask_idx; + + emit MessageSubmitted(idx, inputmask_idx, masked_input); + + // The input masks are deactivated after first use + inputmasks_claimed[inputmask_idx] = address(0); + } + + // ######################### + // 3. Initiate Mixing Epochs + // ######################### + + uint public constant K = 32; // Mix Size + + // Preprocessing requirements + uint public constant PER_MIX_TRIPLES = (K / 2) * 5 * 5; // k log^2 k + uint public constant PER_MIX_BITS = (K / 2) * 5 * 5; + + // Return the maximum number of mixes that can be run with the + // available preprocessing + function mixes_available() public view returns(uint) { + uint triples_available = preprocess.triples - preprocess_used.triples; + uint bits_available = preprocess.bits - preprocess_used.bits; + return min(triples_available / PER_MIX_TRIPLES, + bits_available / PER_MIX_BITS); + } + + // Step 3.a. Trigger a mix to start + uint public inputs_mixed; + uint public epochs_initiated; + event MixingEpochInitiated(uint epoch); + + function inputs_ready() public view returns(uint) { + return input_queue.length - inputs_mixed; + } + + function initiate_mix() public { + // Must mix eactly K values in each epoch + require(input_queue.length >= inputs_mixed + K); + + // Can only initiate mix if enough preprocessings are ready + require(preprocess.triples >= preprocess_used.triples + PER_MIX_TRIPLES); + require(preprocess.bits >= preprocess_used.bits + PER_MIX_BITS); + preprocess_used.triples += PER_MIX_TRIPLES; + preprocess_used.bits += PER_MIX_BITS; + + inputs_mixed += K; + emit MixingEpochInitiated(epochs_initiated); + epochs_initiated += 1; + output_votes.length = epochs_initiated; + output_hashes.length = epochs_initiated; + } + + // Step 3.b. Output reporting: the output is considered "approved" once + // at least t+1 servers report it + + uint public outputs_ready; + event MixOutput(uint epoch, string output); + bytes32[] public output_hashes; + uint[] public output_votes; + mapping (uint => uint) public server_voted; // highest epoch voted in + + function propose_output(uint epoch, string output) public { + require(epoch < epochs_initiated); // can't provide output if it hasn't been initiated + require(servermap[msg.sender] > 0); // only valid servers + uint id = servermap[msg.sender] - 1; + + // Each server can only vote once per epoch + // Hazard note: honest servers must vote in strict ascending order, or votes + // will be lost! + require(epoch <= server_voted[id]); + server_voted[id] = max(epoch + 1, server_voted[id]); + + bytes32 output_hash = sha3(output); + + if (output_votes[epoch] > 0) { + // All the votes must match + require(output_hash == output_hashes[epoch]); + } else { + output_hashes[epoch] = output_hash; + } + + output_votes[epoch] += 1; + if (output_votes[epoch] == t + 1) { // at least one honest node agrees + emit MixOutput(epoch, output); + outputs_ready += 1; + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 3852d0eb..958b4f11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: - ./benchmark:/usr/src/HoneyBadgerMPC/benchmark - ./aws:/usr/src/HoneyBadgerMPC/aws - ./conf:/usr/src/HoneyBadgerMPC/conf + - ./contracts:/usr/src/HoneyBadgerMPC/contracts - ./docs:/usr/src/HoneyBadgerMPC/docs - ./honeybadgermpc:/usr/src/HoneyBadgerMPC/honeybadgermpc - ./scripts:/usr/src/HoneyBadgerMPC/scripts @@ -29,3 +30,6 @@ services: - ./pairing/setup.py:/usr/src/HoneyBadgerMPC/pairing/setup.py - /usr/src/HoneyBadgerMPC/honeybadgermpc/ntl # Directory _not_ mounted from host command: pytest -v --cov=honeybadgermpc + environment: + # FIXME temporary, should be in developer settings + PYTHONBREAKPOINT: ipdb.set_trace diff --git a/docs/conf.py b/docs/conf.py index 994c802c..1a2ad1a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,12 +44,15 @@ "sphinx.ext.autodoc", "sphinx.ext.doctest", "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.mathjax", "sphinx.ext.viewcode", "sphinx_tabs.tabs", "m2r", + "sphinxcontrib.bibtex", + "sphinxcontrib.soliditydomain", ] autodoc_default_options = { @@ -57,7 +60,7 @@ "undoc-members": None, "private-members": None, "inherited-members": None, - "show-inheritance": None, + # "show-inheritance": None, } # Add any paths that contain templates here, relative to this directory. diff --git a/docs/index.rst b/docs/index.rst index f42df0d3..b550110a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,8 +30,9 @@ .. toctree:: :maxdepth: 1 - :caption: Integrations + :caption: Blockchain Integrations + integrations/eth integrations/hyperledger-fabric .. toctree:: diff --git a/docs/integrations/eth.rst b/docs/integrations/eth.rst new file mode 100644 index 00000000..e18470bf --- /dev/null +++ b/docs/integrations/eth.rst @@ -0,0 +1,461 @@ +AsynchroMix with Ethereum as an MPC Coordinator +=============================================== +A blockchain can be used as a coordinating mechanism to run an MPC +program. This document covers the AsynchroMix application. + +In the paper, Reliable Broadcast and Common Subset are used to +"coordinate" the MPC operations. Below is the protocol as it is in the +paper and after that the AsynchroMix protocol is revisited and +presented as it is implemented in :mod:`apps.asynchromix.asynchromix` +where Ethereum is used in place of Reliable Broadcast and Common +Subset. + +AsynchroMix (paper version) +--------------------------- +As presented in the paper :cite:`honeybadgermpc` (`eprint iacr version +`_), figure 5, section 4. + +* Input: Each client :math:`C_j` receives an input :math:`m_j` +* Output: In each epoch a subset of client inputs + :math:`m_1, \ldots, m_k` are selected, and a permutation + :math:`\pi (m_1, \ldots, m_k)` is published where :math:`\pi` does + not depend on the input permutation +* Preprocessing: + + * For each :math:`m_j`, a random :math:`[\![r_j]\!]`, where each + client has received :math:`r_j` + * Preprocessing for PowerMix and/or Switching-Network + +* Protocol (for client :math:`C_j`): + + 1. Set :math:`\overline m_j := m_j + r_j` + 2. :math:`\textsf{ReliableBroadcast} \; \overline m_j` + 3. Wait until :math:`m_j` appears in the output of a mixing epoch + +* Protocol (for server :math:`P_i`) + + - Initialize for each client :math:`C_j` + + .. math:: + :nowrap: + + \begin{align*} + \textsf{input}_j & := 0 \quad \textit{ // No. of inputs received from } C_j \\ + \textsf{done}_j & := 0 \quad \textit{ // No. of messages mixed for } C_j + \end{align*} + + - On receiving :math:`\overline m_j` output from + :math:`\textsf{ReliableBroadcast}` client :math:`C_j` at any time, + set :math:`\textsf{input}_j := \textsf{input}_{j + 1}` + - Proceed in consecutive mixing epochs :math:`e`: + + **Input Collection Phase** + + * Let :math:`b_i` be a :math:`\lvert \mathcal{C} \rvert`-bit vector + where :math:`b_{i,j} = 1` if :math:`\textsf{input}_j \gt + \textsf{done}_j`. + * Pass :math:`b_i` as input to an instance of + :math:`\textsf{CommonSubset}`. + * Wait to receive :math:`b` from :math:`\textsf{CommonSubset}`, where + :math:`b` is an :math:`n \times \lvert \mathcal{C} \rvert` matrix, each row of + :math:`b` corresponds to the input from one server, and at least + :math:`n − t` of the rows are non-default. + * Let :math:`b_{\cdot,, j}` denote the column corresponding to client + :math:`C_j`. + * For each :math:`C_j`, + + .. math:: + :nowrap: + + \begin{equation} + [\![m_j]\!] := + \begin{cases} + \overline m_j - [\![r_j]\!] & \sum b_{\cdot,j} \geq t+1 \\ + 0 & \text{otherwise} + \end{cases} + \end{equation} + + **Online Phase** + + Switch Network Option + + Run the MPC Program switching-network on + :math:`\{[\![m_{j,k_j}]\!]\}`, resulting in + :math:`\pi (m_1, \ldots, m_k)` + Requires :math:`k` rounds, + + Powermix Option + + Run the MPC Program power-mix on + :math:`\{[\![m_{j,k_j}]\!]\}`, resulting in + :math:`\pi (m_1, \ldots, m_k)` + + Set :math:`\textsf{done}_j := \textsf{done}_{j+1}` for each + client :math:`C_j` whose input was mixed this epoch + + +AsynchroMix & Ethereum +---------------------- +In the original protocol asynchronous Reliable Broadcast and Common +Subset are used to orchestrate the different MPC operations that +require consensus amongst the MPC servers. See section 2.3 and 4 of the +paper for details. In this section the original protocol is presented +as it is implemented under :mod:`apps.asynchromix.asynchromix`. In +:mod:`apps.asynchromix.asynchromix` Ethereum is used as a consensus +backbone to orchestrate the MPC operations. + +**Main components:** + +* coordinator: blockchain (:sol:contract:`AsynchromixCoordinator`) +* asynchromix servers ( + :class:`~apps.asynchromix.asynchromix.AsynchromixServer`) +* asynchromix clients ( + :class:`~apps.asynchromix.asynchromix.AsynchromixClient`) + + +Input +^^^^^ +Each client :math:`C_j` receives an input :math:`m_j`. + +Currently, only one client is used, and the client itself sends a +series of "dummy" messages. In +:func:`~apps.asynchromix.asynchromix.AsynchromixClient._run()`: + +.. code-block:: python + + class AsynchromixClient: + + async def _run(self): + + # ... + for epoch in range(1000): + receipts = [] + for i in range(32): + m = f"message:{epoch}:{i}" + task = asyncio.ensure_future(self.send_message(m)) + receipts.append(task) + receipts = await asyncio.gather(*receipts) + # ... + +Output +^^^^^^ +In each epoch a subset of client inputs :math:`m_1, \ldots, m_k` are +selected, and a permutation :math:`\pi (m_1, \ldots, m_k)` is published +where :math:`\pi` does not depend on the input permutation + +Preprocessing +^^^^^^^^^^^^^ +* For each :math:`m_j`, a random :math:`[\![r_j]\!]`, where each client + has received :math:`r_j` +* Preprocessing for PowerMix and/or Switching-Network + +.. note:: At the moment the MPC program uses the switching network ( + :func:`~apps.asynchromix.butterfly_network.iterated_butterfly_network`). + +.. todo:: Explain how the preprocessing values are generated. + +.. todo:: Explain what preprocessing is done for the switching + (butterfly) network. + +In the :mod:`~apps.asynchromix.asynchromix` example the client ( +:class:`~apps.asynchromix.asynchromix.AsynchromixClient`) + +1. waits for an input mask to be ready via the smart contract function + :sol:func:`inputmasks_available`; +2. reserves an input mask via :sol:func:`reserve_inputmask`; +3. fetches the input mask from the servers (the client reconstructs the + input mask, given sufficient shares from the servers) + +Below are some code snippets that perform the above 3 steps. *Some +details of the implementation are omitted in order to ease the +presentation.* + +.. code-block:: python + + class AsynchromixClient: + + async def send_message(self, m): + contract_concise = ConciseContract(self.contract) + + # Step 1. Wait until there is input available, and enough triples + while True: + inputmasks_available = contract_concise.inputmasks_available() + if inputmasks_available >= 1: + break + await asyncio.sleep(5) + + # Step 2. Reserve the input mask + tx_hash = self.contract.functions.reserve_inputmask().transact( + {"from": self.w3.eth.accounts[0]} + ) + tx_receipt = await wait_for_receipt(self.w3, tx_hash) + rich_logs = self.contract.events.InputMaskClaimed().processReceipt(tx_receipt) + inputmask_idx = rich_logs[0]["args"]["inputmask_idx"] + + # Step 3. Fetch the input mask from the servers + inputmask = await self._get_inputmask(inputmask_idx) + + async def _get_inputmask(self, idx): + contract_concise = ConciseContract(self.contract) + n = contract_concise.n() + poly = polynomials_over(field) + eval_point = EvalPoint(field, n, use_omega_powers=False) + shares = [] + for i in range(n): + share = self.req_mask(i, idx) + shares.append(share) + shares = await asyncio.gather(*shares) + shares = [(eval_point(i), share) for i, share in enumerate(shares)] + mask = poly.interpolate_at(shares, 0) + return mask + + +AsynchromixClient Protocol (for client :math:`C_j`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +1. Set :math:`\overline m_j := m_j + r_j` +2. :math:`\textsf{ReliableBroadcast} \; \overline m_j` +3. Wait until :math:`m_j` appears in the output of a mixing epoch + +For step 2, instead of using :math:`\textsf{ReliableBroadcast}` the +client (:class:`~apps.asynchromix.asynchromix.AsynchromixClient`) +publishes the masked message :math:`\overline m_j` onto the Ethereum +blockchain via the smart contract function :sol:func:`submit_message`. +The masked messages are stored in the +:sol:contract:`AsynchromixCoordinator` contract' state variable +:sol:svar:`input_queue`. + +.. code-block:: python + + class AsynchromixClient: + + async def send_message(self, m): + # ... + masked_message = message + inputmask + masked_message_bytes = self.w3.toBytes(hexstr=hex(masked_message.value)) + masked_message_bytes = masked_message_bytes.rjust(32, b"\x00") + + # Step 4. Publish the masked input + tx_hash = self.contract.functions.submit_message( + inputmask_idx, masked_message_bytes + ).transact({"from": self.w3.eth.accounts[0]}) + tx_receipt = await wait_for_receipt(self.w3, tx_hash) + + +AsynchromixServer Protocol (for server :math:`P_i`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Initialize for each client :math:`C_j` + +.. math:: + :nowrap: + + \begin{align*} + \textsf{input}_j & := 0 \quad \textit{ // No. of inputs received from } C_j \\ + \textsf{done}_j & := 0 \quad \textit{ // No. of messages mixed for } C_j + \end{align*} + +.. todo:: Is there a :math:`\textsf{done}_j` state variable in the + code or contract? + +- On receiving :math:`\overline m_j` output from + :math:`\textsf{ReliableBroadcast}` client :math:`C_j` at any time, + set :math:`\textsf{input}_j := \textsf{input}_{j + 1}` + +This step is handled by the smart contract function +:sol:func:`submit_message`. When the client submits a masked message, +the masked input (message) is stored in the contract's state variable +:sol:svar:`input_queue` and the length of the input queue ( +:math:`\textsf{input}_j`) is incremented by one. + +.. code-block:: solidity + + struct Input { + bytes32 masked_input; // (m+r) + uint inputmask; // index in inputmask of mask [r] + } + + Input[] public input_queue; // All inputs sent so far + + event MessageSubmitted(uint idx, uint inputmask_idx, bytes32 masked_input); + + function submit_message(uint inputmask_idx, bytes32 masked_input) public { + // Must be authorized to use this input mask + require(inputmasks_claimed[inputmask_idx] == msg.sender); + + uint idx = input_queue.length; + input_queue.length += 1; + + input_queue[idx].masked_input = masked_input; + input_queue[idx].inputmask = inputmask_idx; + + emit MessageSubmitted(idx, inputmask_idx, masked_input); + + // The input masks are deactivated after first use + inputmasks_claimed[inputmask_idx] = address(0); + } + +- Proceed in consecutive mixing epochs :math:`e`: + + **Input Collection Phase** + + * Let :math:`b_i` be a :math:`|\mathcal{C}|`-bit vector where + :math:`b_{i,j} = 1` if :math:`\textsf{input}_j \gt + \textsf{done}_j`. + * Pass :math:`b_i` as input to an instance of + :math:`\textsf{CommonSubset}`. + * Wait to receive :math:`b` from :math:`\textsf{CommonSubset}`, where + :math:`b` is an :math:`n \times |\mathcal{C}|` matrix, each row of + :math:`b` corresponds to the input from one server, and at least + :math:`n − t` of the rows are non-default. + * Let :math:`b_{\cdot,, j}` denote the column corresponding to client + :math:`C_j`. + * For each :math:`C_j`, + + .. math:: + :nowrap: + + \begin{equation} + [\![m_j]\!] := + \begin{cases} + \overline m_j - [\![r_j]\!] & \sum b_{\cdot,j} \geq t+1 \\ + 0 & \text{otherwise} + \end{cases} + \end{equation} + + .. todo:: Explain how the contract function :sol:func:`propose_output` + is used instead by the servers to submit their shuffled messages + :math:`\pi (m_1, \ldots, m_k)` that were obtained in the MPC run + for the epoch. + + **Online Phase** + + Switch Network Option + + Run the MPC Program switching-network on + :math:`\{[\![m_{j,k_j}]\!]\}`, resulting in + :math:`\pi (m_1, \ldots, m_k)` + Requires :math:`k` rounds, + + Powermix Option + + Run the MPC Program power-mix on + :math:`\{[\![m_{j,k_j}]\!]\}`, resulting in + :math:`\pi (m_1, \ldots, m_k)` + + Set :math:`\textsf{done}_j := \textsf{done}_{j+1}` for each + client :math:`C_j` whose input was mixed this epoch + + .. todo:: Explain `briefly` when, where (in the code), and how the + messages are shuffled via the switching (butterfly) network + in the MPC program. + + Also, is there a :math:`\textsf{done}_j` state variable in the + code or contract? + +Walkthrough +----------- +This section presents a step-by-step walkthrough of the code involved +to run the asynchromix example. + +To run the example: + +.. code-block:: shell + + $ python apps/asynchromix/asynchromix.py + + +So what happens when the above command is run? + +1. :py:func:`~apps.asynchromix.asynchromix.test_asynchromix` is run. +2. :py:func:`~apps.asynchromix.asynchromix.test_asynchromix` takes care + of running a local test Ethereum blockchain using `Ganache`_ and of + starting the main loop via + :py:func:`~apps.asynchromix.asynchromix.run_eth()`. More precisely, + :py:func:`~apps.asynchromix.asynchromix.test_asynchromix` runs the + command: + + .. code-block:: shell + + ganache-cli -p 8545 -a 50 -b 1 > acctKeys.json 2>&1 + + in a subprocess, in a :py:func:`contextmanager` ( + :py:func:`~apps.asynchromix.asynchromix.run_and_terminate_process`) + and within this context, in which Ethereum is running, the function + :py:func:`~apps.asynchromix.asynchromix.run_eth()` is invoked. +3. :py:func:`~apps.asynchromix.asynchromix.run_eth()` takes care of + instantiating a connection to the local Ethereum node: + + .. code-block:: python + + w3 = Web3(HTTPProvider()) + + and of starting the main loop which needs a connection to Ethereum: + + .. code-block:: python + + loop.run_until_complete(asyncio.gather(main_loop(w3))) +4. The :py:func:`~apps.asynchromix.asynchromix.main_loop` takes care of + four main things: + + 1. creating a coordinator contract (and web3 interface to it); + 2. instantiating the asynchromix servers; + 3. instantiating an asynchromix client; + 4. starting the servers and client and waiting for the completion of + their tasks. + +Initialization Phase +-------------------- +.. todo:: This section's goal is to outline the basic setup + requirements such as: + + * eth accounts creation for the MPC servers; + * "loading" of the contract on chain. + + + +Internal API docs +----------------- + +Asynchromix Coordinator Contract +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. .. sol:contract:: AsynchromixCoordinator +.. +.. .. sol:function:: inputmasks_available () public view returns (uint) +.. +.. Returns the number of input masks that are available. + +.. autosolcontract:: AsynchromixCoordinator + + + +Asynchromix Servers +^^^^^^^^^^^^^^^^^^^ +.. autoclass:: apps.asynchromix.asynchromix.AsynchromixServer + +.. automodule:: apps.asynchromix.butterfly_network + +Asynchromix Client +^^^^^^^^^^^^^^^^^^ +.. autoclass:: apps.asynchromix.asynchromix.AsynchromixClient + + + + +.. .. automodule:: apps.asynchromix.asynchromix + + +Questions +--------- +When submitting a message to Ethereum, via the contract, is the +identity of the client public? Can it be kept hidden? + +What about intersection attacks? + + +References +---------- +.. bibliography:: refs.bib + + +.. _paper: https://eprint.iacr.org/2019/883.pdf +.. _Ganache: https://github.com/trufflesuite/ganache diff --git a/docs/integrations/refs.bib b/docs/integrations/refs.bib new file mode 100644 index 00000000..5fd7d3a7 --- /dev/null +++ b/docs/integrations/refs.bib @@ -0,0 +1,16 @@ +@inproceedings{honeybadgermpc, +author = {Lu, Donghang and Yurek, Thomas and Kulshreshtha, Samarth and Govind, Rahul and Kate, Aniket and Miller, Andrew}, +title = {"HoneyBadgerMPC and AsynchroMix: Practical Asynchronous MPC and Its Application to Anonymous Communication"}, +year = {2019}, +isbn = {9781450367479}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +url = {https://doi.org/10.1145/3319535.3354238}, +doi = {10.1145/3319535.3354238}, +booktitle = {Proceedings of the 2019 ACM SIGSAC Conference on Computer and Communications Security}, +pages = {887–903}, +numpages = {17}, +keywords = {robustness, anonymous communication, asynchronous mixing, fairness, honeybadgerMPC}, +location = {London, United Kingdom}, +series = {CCS ’19} +} diff --git a/honeybadgermpc/preprocessing.py b/honeybadgermpc/preprocessing.py index bd2d897a..62f59d8a 100644 --- a/honeybadgermpc/preprocessing.py +++ b/honeybadgermpc/preprocessing.py @@ -17,6 +17,10 @@ from .ntl import vandermonde_batch_evaluate from .polynomial import polynomials_over +logger = logging.getLogger(__name__) +# FIXME move log level setting to an entrypoint (e.g. __init__ or an app main entry) +logger.setLevel(os.environ.get("HBMPC_LOGLEVEL", logging.INFO)) + class PreProcessingConstants(Enum): SHARED_DATA_DIR = "sharedata/" @@ -99,7 +103,10 @@ def get_value(self, context, *args, **kwargs): key = (context.myid, context.N, context.t) to_return, used = self._get_value(context, key, *args, **kwargs) + logger.debug(f'got value "{to_return}" and used "{used}"') + logger.debug(f"decrement count by {used}") self.count[key] -= used + logger.debug(f"count is now: {self.count}") return to_return @@ -123,7 +130,7 @@ def _read_preprocessing_file(self, file_name): return values[3:] def _write_preprocessing_file( - self, file_name, degree, context_id, values, append=False + self, file_name, degree, context_id, values, append=False, refresh_cache=False ): """ Write the values to the preprocessing file given by the filename. When append is true, this will append to an existing file, otherwise, it will @@ -148,6 +155,8 @@ def _write_preprocessing_file( print(*values, file=f, sep="\n") f.close() + if refresh_cache: + self._refresh_cache() def build_filename(self, n, t, context_id, prefix=None): """ Given a file prefix, and metadata, return the filename to put @@ -189,6 +198,10 @@ def _refresh_cache(self): """ Refreshes the cache by reading in sharedata files, and updating the cache values and count variables. """ + logger.debug(f"(- {self.preprocessing_name} -) refreshing cache") + logger.debug( + f"(- {self.preprocessing_name} -) before cache refresh, count is: {dict(self.count)}" + ) self.cache = defaultdict(chain) self.count = defaultdict(int) @@ -208,6 +221,10 @@ def _refresh_cache(self): self.cache[key] = chain(values) self.count[key] = len(values) + logger.debug( + f"(- {self.preprocessing_name} -) after cache refresh, count is: {dict(self.count)}" + ) + def _write_polys(self, n, t, polys, append=False, prefix=None): """ Given a file prefix, a list of polynomials, and associated n, t values, write the preprocessing for the share values represented by the polnomials. @@ -410,7 +427,7 @@ def _generate_polys(self, k, n, t): def _get_value(self, context, key, t=None): t = t if t is not None else context.t - assert self.count[key] >= 1 + assert self.count[key] >= 1, f"key is: {key}\ncount is: {self.count}\n" return context.Share(next(self.cache[key]), t), 1 @@ -426,9 +443,12 @@ def _get_value(self, context, key): assert self.count[key] >= self._preprocessing_stride, ( f"Expected " f"{self._preprocessing_stride} elements of {self.preprocessing_name}, " - f"but found only {self.count[key]}" + f"but found only {self.count[key]}\n" + f"key is: {key}\n" + f"count is: {self.count}\n" ) + logger.debug("getting value ...") values = tuple( context.Share(next(self.cache[key])) for _ in range(self._preprocessing_stride) @@ -572,11 +592,12 @@ def _init_data_dir(self): def clear_preprocessing(self): """ Delete all things from the preprocessing folder """ + logger.debug( + f"Deleting all files from preprocessing folder: {self.data_directory}" + ) rmtree( self.data_directory, - onerror=lambda f, p, e: logging.debug( - f"Error deleting data directory: {e}" - ), + onerror=lambda f, p, e: logger.debug(f"Error deleting data directory: {e}"), ) self._init_data_dir() @@ -585,7 +606,7 @@ async def wait_for_preprocessing(self, timeout=1): """ Block until the ready file is created """ while not os.path.exists(self._ready_file): - logging.info(f"waiting for preprocessing {self._ready_file}") + logger.debug(f"waiting for preprocessing {self._ready_file}") await asyncio.sleep(timeout) def preprocessing_done(self): diff --git a/setup.py b/setup.py index e4ecafaa..c2212a1e 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,8 @@ DOCS_REQUIRE = [ "Sphinx", "sphinx-autobuild", + "sphinxcontrib-bibtex", + "sphinxcontrib-soliditydomain", "sphinx_rtd_theme", "sphinx_tabs", "m2r",