diff --git a/.circleci/config.yml b/.circleci/config.yml index 34995d47..e0a4695b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,11 @@ jobs: name: install dependencies # The "--editable" option is needed until we resolve issue #377. # See https://github.com/initc3/HoneyBadgerMPC/issues/377 for more info. - command: mkdir -p /var/log/hbmpc && pip install --editable .[docs] + command: | + make -C apps/asynchromix/cpp + pip install -v pairing/ + mkdir -p /var/log/hbmpc + pip install --editable .[docs] - save_cache: paths: diff --git a/.gitignore b/.gitignore index 286ac8fc..a01a595d 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,5 @@ sharedata/ .ci/deploy_key .ci/deploy_key.pub +# Intellij +.idea/ diff --git a/Makefile b/Makefile index 399c6596..04cd0584 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,8 @@ .DEFAULT_GOAL := help define BROWSER_PYSCRIPT -import os, webbrowser, sys -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url +import sys, webbrowser webbrowser.open("http://localhost:58888/" + sys.argv[1]) endef @@ -22,7 +18,9 @@ for line in sys.stdin: target, help = match.groups() print("%-20s %s" % (target, help)) endef + export PRINT_HELP_PYSCRIPT + BROWSER := python -c "$$BROWSER_PYSCRIPT" help: diff --git a/apps/auctions/volume_matching.py b/apps/auctions/volume_matching.py new file mode 100644 index 00000000..ef885938 --- /dev/null +++ b/apps/auctions/volume_matching.py @@ -0,0 +1,346 @@ +# Original code was written by Harjasleen Malvai + +""" +In volume matching auction, buy and sell orders are matched only on volume +while price is determined by reference to some external market. +""" + +import asyncio +import logging + +from honeybadgermpc.mpc import TaskProgramRunner +from honeybadgermpc.preprocessing import ( + PreProcessedElements as FakePreProcessedElements, +) +from honeybadgermpc.progs.fixedpoint import FixedPoint +from honeybadgermpc.progs.mixins.share_arithmetic import ( + BeaverMultiply, + BeaverMultiplyArrays, + MixinConstants, +) + +config = { + MixinConstants.MultiplyShareArray: BeaverMultiplyArrays(), + MixinConstants.MultiplyShare: BeaverMultiply(), +} + + +async def compute_bids(ctx, balances, bids, price): + """Compute all valid bids for each user. + According to current market price, a bid becomes invalid + if the user doesn't have enough balance to pay for this bid. + We only keep valid bids and classify them into + buy bids(volume > 0) and sell bids(volume < 0). + Invalid bids will be discarded after executing this function. + + Parameters + ---------- + balances : dict of dict + Dictionary of the form: ``{address: {cointype: balance}}``. + Address is a string representing + the public address of the user. + Cointype is a string. + Now we only support two cointypes, 'eth' and 'erc20'. + Balance is a FixedPoint number. + bids : list of tuple + A list of bids. + Each bid is a tuple of two elements ``(address, volume)``, + the address of the owner and the volume of this bid. + Address is a string and volume is a FixedPoint number. + When volume is larger than zero, this bid is a buy bid, + which means the owner wants to buy 'volume' units of tokens + with 'volume * price' units of ETH. + When volume is less than zero, the bid is a sell bid, + which means the owner wants to sell 'volume' units of tokens + for 'volume * price' units of ETH. + price : FixedPoint + In volume matching, price is determined by reference + to some external lit market. Price is how many units of ETH + have the same value as one unit of token. + + Returns + ------- + buys : list of tuple + List of valid buy orders ``(address, volume)``. + sells : list of tuple + List of valid sell orders ``(address, volume)``. + Since we have separated buy and sell bids, + now every sell bid has volume larger than zero. + """ + + one = ctx.Share(1) + zero = ctx.Share(0) + fp_zero = FixedPoint(ctx, zero) + + used_balances = {} + # TODO for key in balances: + for key in balances.keys(): + used_balances[key] = { + "eth": create_clear_share(ctx, 0), + "erc20": create_clear_share(ctx, 0), + } + + buys = [] + sells = [] + + for bid in bids: + addr, vol = bid + + is_sell = await vol.ltz() + is_buy = one - is_sell + + sell_vol = fp_zero - FixedPoint(ctx, await (is_sell * vol.share)) + buy_vol = FixedPoint(ctx, await (is_buy * vol.share)) + + is_sell_vol_valid = one - await balances[addr]["erc20"].lt( + sell_vol + used_balances[addr]["erc20"] + ) + is_buy_vol_valid = one - await balances[addr]["eth"].lt( + (await buy_vol.__mul__(price)) + used_balances[addr]["eth"] + ) + + fp_is_sell_vol_valid = FixedPoint(ctx, is_sell_vol_valid * 2 ** 32) + fp_is_buy_vol_valid = FixedPoint(ctx, is_buy_vol_valid * 2 ** 32) + + sell_vol = await fp_is_sell_vol_valid.__mul__(sell_vol) + buy_vol = await fp_is_buy_vol_valid.__mul__(buy_vol) + + sells.append((addr, sell_vol)) + buys.append((addr, buy_vol)) + + used_balances[addr]["erc20"] = used_balances[addr]["erc20"] + sell_vol + used_balances[addr]["eth"] = used_balances[addr]["eth"] + ( + await buy_vol.__mul__(price) + ) + + return buys, sells + + +async def volume_matching(ctx, buys, sells): + """Given all valid buy and sell bids, + this function runs the volume matching algorithm, + where buy and sell bids are matched only on volume + with no price information considered. + First we compute the total amount to be matched, + i.e., the smaller one between total buy volume + and total sell volume. + Then we match for buy bids and sell bids respectively. + After matching, each bid is split into matched and rest part. + This function returns four lists of bids, + where matched_buys + res_buys = buys and + matched_sells + res_sells = sells. + + Parameters + ---------- + buys : list of tuple + List of valid buy orders. An order is a 2-tuple: + ``(address, volume)``. + sells : list of tuple + List of valid sell orders. An order is a 2-tuple: + ``(address, volume)``. + + Returns + ------- + matched_buys : list of tuple + The matched part of each buy orders. + matched_sells : list of tuple + The matched part of each sell orders. + res_buys : list of tuple + The unmatched part of each buy orders. + res_sells : list of tuple + The unmatched part of each sell orders. + """ + + zero = ctx.Share(0) + fp_zero = FixedPoint(ctx, zero) + + # compute total amount of volume to be matched + matched_buys, matched_sells = [], [] + res_buys, res_sells = [], [] + + total_sells = fp_zero + for sell in sells: + total_sells = total_sells + sell[1] + + total_buys = fp_zero + for buy in buys: + total_buys = total_buys + buy[1] + + f = await total_buys.lt(total_sells) + fp_f = FixedPoint(ctx, f * 2 ** 32) + matching_volume = (await (total_buys - total_sells).__mul__(fp_f)) + total_sells + + # match for sell bids + rest_volume = matching_volume + for sell in sells: + addr, sell_vol = sell + + z1 = await fp_zero.lt(rest_volume) + fp_z1 = FixedPoint(ctx, z1 * 2 ** 32) + z2 = await rest_volume.lt(sell_vol) + fp_z2 = FixedPoint(ctx, z2 * 2 ** 32) + + matched_vol = await ( + (await (rest_volume - sell_vol).__mul__(fp_z2)) + sell_vol + ).__mul__(fp_z1) + rest_volume = rest_volume - matched_vol + + matched_sells.append([addr, matched_vol]) + res_sells.append([addr, sell_vol - matched_vol]) + + # match for buy bids + rest_volume = matching_volume + for buy in buys: + addr, buy_vol = buy + + z1 = await fp_zero.lt(rest_volume) + fp_z1 = FixedPoint(ctx, z1 * 2 ** 32) + z2 = await rest_volume.lt(buy_vol) + fp_z2 = FixedPoint(ctx, z2 * 2 ** 32) + + matched_vol = await ( + (await (rest_volume - buy_vol).__mul__(fp_z2)) + buy_vol + ).__mul__(fp_z1) + rest_volume = rest_volume - matched_vol + + matched_buys.append([addr, matched_vol]) + res_buys.append([addr, buy_vol - matched_vol]) + + return matched_buys, matched_sells, res_buys, res_sells + + +async def compute_new_balances(balances, matched_buys, matched_sells, price): + """Update balances for each user after volume matching + + Parameters + ---------- + balances : dict of dict + Balances of users before matching. + The dict is of the form: ``{address: {cointype: balance}}``, + same as in the :func:`compute_bids` function. + matched_buys : list of tuple + List of matched buy bids. + matched_sells : list of tuple + List of matched sell bids. + price: FixedPoint + External market price + + Returns + ------- + balances : dict of dict + Updated balances after matching. + """ + + for sell in matched_sells: + addr, vol = sell + + balances[addr]["erc20"] = balances[addr]["erc20"] - vol + balances[addr]["eth"] = balances[addr]["eth"] + (await vol.__mul__(price)) + + for buy in matched_buys: + addr, vol = buy + + balances[addr]["eth"] = balances[addr]["eth"] - (await vol.__mul__(price)) + balances[addr]["erc20"] = balances[addr]["erc20"] + vol + + return balances + + +def create_secret_share(ctx, x): + return FixedPoint(ctx, ctx.Share(x * 2 ** 32) + ctx.preproc.get_zero(ctx)) + + +def create_clear_share(ctx, x): + return FixedPoint(ctx, ctx.Share(x * 2 ** 32)) + + +async def prog(ctx): + ctx.preproc = FakePreProcessedElements() + + price = create_clear_share(ctx, 3) + + balances = {} + balances["0x125"] = { + "eth": create_clear_share(ctx, 15), + "erc20": create_clear_share(ctx, 1), + } + balances["0x127"] = { + "eth": create_clear_share(ctx, 18), + "erc20": create_clear_share(ctx, 0), + } + balances["0x128"] = { + "eth": create_clear_share(ctx, 2), + "erc20": create_clear_share(ctx, 3), + } + balances["0x129"] = { + "eth": create_clear_share(ctx, 0), + "erc20": create_clear_share(ctx, 15), + } + + _balances = [ + (await x["eth"].open(), await x["erc20"].open()) for x in balances.values() + ] + logging.info(f"balances initial {_balances}") + + bids = [] + bids.append(["0x125", create_secret_share(ctx, 5)]) + bids.append(["0x127", create_secret_share(ctx, 7)]) + bids.append(["0x128", create_secret_share(ctx, -3)]) + bids.append(["0x129", create_secret_share(ctx, -11)]) + buys, sells = await compute_bids(ctx, balances, bids, price) + + _buys = [await x[1].open() for x in buys] + _sells = [await x[1].open() for x in sells] + logging.info(f"buys initial: {_buys} and sells initial: {_sells}") + + matched_buys, matched_sells, res_buys, res_sells = await volume_matching( + ctx, buys, sells + ) + + _matched_buys = [await x[1].open() for x in matched_buys] + _matched_sells = [await x[1].open() for x in matched_sells] + logging.info(f"buys matched: {_matched_buys} and sells matched: {_matched_sells}") + + _res_buys = [await x[1].open() for x in res_buys] + _res_sells = [await x[1].open() for x in res_sells] + logging.info(f"buys rest: {_res_buys} and sells rest: {_res_sells}") + + _balances = [ + (await x["eth"].open(), await x["erc20"].open()) for x in balances.values() + ] + logging.info(f"balances rest {_balances}") + + final_balances = await compute_new_balances( + balances, matched_buys, matched_sells, price + ) + + _final_balances = [ + (await x["eth"].open(), await x["erc20"].open()) + for x in final_balances.values() + ] + logging.info(f"balances rest {_final_balances}") + + logging.info(f"[{ctx.myid}] done") + + +async def dark_pewl(): + n, t = 4, 1 + k = 10000 + pp = FakePreProcessedElements() + pp.generate_zeros(k, n, t) + pp.generate_triples(k, n, t) + pp.generate_bits(k, n, t) + program_runner = TaskProgramRunner(n, t, config) + program_runner.add(prog) + results = await program_runner.join() + return results + + +def main(): + asyncio.set_event_loop(asyncio.new_event_loop()) + loop = asyncio.get_event_loop() + loop.run_until_complete(dark_pewl()) + + +if __name__ == "__main__": + main() diff --git a/docs.yml b/docs.yml index 4ec16ae1..f96d7417 100644 --- a/docs.yml +++ b/docs.yml @@ -8,6 +8,7 @@ services: dockerfile: Dockerfile volumes: - .:/usr/src/HoneyBadgerMPC + - /usr/src/HoneyBadgerMPC/honeybadgermpc/ntl # Directory _not_ mounted from host environment: - O=-W --keep-going command: make -C docs html diff --git a/docs/apps/auctions.rst b/docs/apps/auctions.rst new file mode 100644 index 00000000..1304c778 --- /dev/null +++ b/docs/apps/auctions.rst @@ -0,0 +1,7 @@ +Auctions +======== + +Volume Matching +--------------- + +.. automodule:: apps.auctions.volume_matching diff --git a/docs/conf.py b/docs/conf.py index a286bb22..5e872643 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,6 +47,7 @@ "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.mathjax", + "sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx_tabs.tabs", "recommonmark", diff --git a/docs/index.rst b/docs/index.rst index 4de35a7a..906c378c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,12 @@ .. source: https://docs.python-guide.org/writing/documentation/#project-publication +.. toctree:: + :maxdepth: 1 + :caption: Apps + + apps/auctions + .. toctree:: :maxdepth: 1 :caption: Integrations diff --git a/tests/test_volume_matching.py b/tests/test_volume_matching.py new file mode 100644 index 00000000..9626ef63 --- /dev/null +++ b/tests/test_volume_matching.py @@ -0,0 +1,178 @@ +import logging +import random + +from pytest import mark + +from apps.auctions.volume_matching import ( + compute_bids, + compute_new_balances, + create_clear_share, + create_secret_share, + volume_matching, +) + +from honeybadgermpc.preprocessing import ( + PreProcessedElements as FakePreProcessedElements, +) +from honeybadgermpc.progs.mixins.share_arithmetic import ( + BeaverMultiply, + BeaverMultiplyArrays, +) + +STANDARD_ARITHMETIC_MIXINS = [BeaverMultiplyArrays(), BeaverMultiply()] + +PREPROCESSING = ["triples", "zeros", "bits"] +n, t = 3, 1 +k = 10000 + + +@mark.asyncio +async def test_volume_matching(test_preprocessing, test_runner): + async def _prog(ctx): + ctx.preproc = FakePreProcessedElements() + + bids = [] + for i in range(num_bids): + bids.append([_bids[i][0], create_secret_share(ctx, _bids[i][1])]) + + balances = {} + for addr in addrs: + balances[addr] = { + "eth": create_clear_share(ctx, bal[addr][0]), + "erc20": create_clear_share(ctx, bal[addr][1]), + } + + _balances = {} + for key, x in balances.items(): + _balances[key] = [await x["eth"].open(), await x["erc20"].open()] + logging.info(f"balances {_balances}") + + price = create_clear_share(ctx, p) + + buys, sells = await compute_bids(ctx, balances, bids, price) + + _buys = [await x[1].open() for x in buys] + _sells = [await x[1].open() for x in sells] + logging.info(f"buys initial: {_buys} and sells initial: {_sells}") + + for i in range(num_bids): + assert _buys[i] == b[i] + assert _sells[i] == s[i] + + matched_buys, matched_sells, res_buys, res_sells = await volume_matching( + ctx, buys, sells + ) + + _matched_buys = [await x[1].open() for x in matched_buys] + _matched_sells = [await x[1].open() for x in matched_sells] + logging.info( + f"buys matched: {_matched_buys} and sells matched: {_matched_sells}" + ) + + _res_buys = [await x[1].open() for x in res_buys] + _res_sells = [await x[1].open() for x in res_sells] + logging.info(f"buys rest: {_res_buys} and sells rest: {_res_sells}") + + for i in range(num_bids): + assert _matched_buys[i] + _res_buys[i] == _buys[i] + assert _matched_sells[i] + _res_sells[i] == _sells[i] + assert _matched_buys[i] == m_buy[i] + assert _matched_sells[i] == m_sell[i] + + res_balances = await compute_new_balances( + balances, matched_buys, matched_sells, price + ) + + _res_balances = {} + for key, x in res_balances.items(): + _res_balances[key] = [await x["eth"].open(), await x["erc20"].open()] + logging.info(f"balances rest {_res_balances}") + + for addr in addrs: + assert final_balances[addr] == _res_balances[addr] + + # random sample price, users addresses, initial balances and bids + p = random.randint(1, 3) + logging.info(f"_price: {p}") + + num_addrs = random.randint(1, 2) + logging.info(f"num_addrs: {num_addrs}") + + addrs = [] + bal = {} + for i in range(num_addrs): + addrs.append(f"0x{i + 120}") + bal[addrs[i]] = [random.randint(10, 40) * p, random.randint(10, 40)] + logging.info(f"addrs: {addrs}") + logging.info(f"bal: {bal}") + + num_bids = random.randint(2, 3) + logging.info(f"number of bids: {num_bids}") + _bids = [] + for i in range(num_bids): + _bids.append([addrs[random.randint(1, num_addrs) - 1], random.randint(-30, 30)]) + logging.info(f"_bids: {_bids}") + + # compute valid buy bids b and sell bids s + sum_buy = 0 + sum_sell = 0 + b = [] + s = [] + used_balances = {} + for addr in addrs: + used_balances[addr] = [0, 0] + + for i in range(num_bids): + b.append(0) + s.append(0) + + addr, vol = _bids[i] + if vol > 0: # buy + if vol * p + used_balances[addr][0] <= bal[addr][0]: + b[i] = vol + used_balances[addr][0] += vol * p + sum_buy += vol + else: # sell + if used_balances[addr][1] - vol <= bal[addr][1]: + s[i] = -vol + used_balances[addr][1] -= vol + sum_sell -= vol + logging.info(f"b: {b}") + logging.info(f"s: {s}") + + # compute matched buy bids m_buy and matched sell bids m_sell + matched = min(sum_sell, sum_buy) + m_buy = [] + for i in range(num_bids): + vol = min(matched, b[i]) + m_buy.append(vol) + matched -= vol + matched = min(sum_sell, sum_buy) + m_sell = [] + for i in range(num_bids): + vol = min(matched, s[i]) + m_sell.append(vol) + matched -= vol + logging.info(f"matched buy: {m_buy}\nmatched sell: {m_sell}") + + # compute balances after matching final_balances + sum_eth = 0 + sum_erc = 0 + for addr in addrs: + sum_eth += bal[addr][0] + sum_erc += bal[addr][1] + logging.info(f"sum eth: {sum_eth} sum erc: {sum_erc}") + + final_balances = {} + for addr in addrs: + final_balances[addr] = [bal[addr][0], bal[addr][1]] + for i in range(num_bids): + addr = _bids[i][0] + final_balances[addr][0] -= p * m_buy[i] + final_balances[addr][1] += m_buy[i] + + final_balances[addr][1] -= m_sell[i] + final_balances[addr][0] += p * m_sell[i] + logging.info(f"final balances: {final_balances}") + + await test_runner(_prog, n, t, PREPROCESSING, k, STANDARD_ARITHMETIC_MIXINS)