diff --git a/jmdaemon/jmdaemon/lnonion.py b/jmdaemon/jmdaemon/lnonion.py index 3f5962f37..9efd55cae 100644 --- a/jmdaemon/jmdaemon/lnonion.py +++ b/jmdaemon/jmdaemon/lnonion.py @@ -10,6 +10,12 @@ from twisted.protocols.basic import LineReceiver log = get_log() +""" +Messaging protocol (which wraps the underlying Joinmarket +messaging protocol) used here is documented in: +Joinmarket-Docs/lightning-messaging.md +""" + LOCAL_CONTROL_MESSAGE_TYPES = {"connect": 785, "disconnect": 787, "connect-in": 797} CONTROL_MESSAGE_TYPES = {"peerlist": 789, "getpeerlist": 791, "handshake": 793, "dn-handshake": 795} @@ -35,193 +41,14 @@ "proto-ver-max": JM_VERSION, "features": {}, "accepted": False, - "nick": "" + "nick": "", + "motd": "Default MOTD, replace with information for the directory." } # states that keep track of relationship to a peer PEER_STATUS_UNCONNECTED, PEER_STATUS_CONNECTED, PEER_STATUS_HANDSHAKED, \ PEER_STATUS_DISCONNECTED = range(4) -""" -### MESSAGE FORMAT USED on the LN-ONION CHANNELS - -( || means concatenation for strings, here) - -Messages conveyed between directly connected nodes with `sendcustommsg` -have the format described here: - -https://lightning.readthedocs.io/PLUGINS.html#custommsg - -Note in particular, that `type` is a two byte hex-encoded string -that prepends the actual message (also hex-encoded), without any intervening -length field. This `type` is the integer specified in this file as *_MESSAGE_TYPES. - -Note also that the type *must* be odd for custom messages (the "it's OK to be odd" principle). - -In text, the messages we send via this mechanism are of this format: - -from-nick || COMMAND_PREFIX || to-nick || COMMAND_PREFIX || cmd || " " || innermessage - -(COMMAND_PREFIX defined in this package's protocol.py) - -Here `innermessage` may be a list of messages (e.g. the multiple offer case) separated by COMMAND PREFIX. - -Note that this syntax will still be as was described here: - -https://github.com/JoinMarket-Org/JoinMarket-Docs/blob/master/Joinmarket-messaging-protocol.md#joinmarket-messaging-protocol - -Note also that there is no chunking requirement applied here, based on the assumption that we have a sufficient 1300 byte limit. -That'll probably be changed later. - -### CONTROL MESSAGES - -#### HANDSHAKE CONTROL MESSAGES - -The message `handshake` is sent by any peer/node not configured to act as -directory node, to any other node it connects to, as the first message. -The message `dn-handshake` is sent by any peer/node which is configured to -act as a directory node, to any other node, as a response to the initial -`handshake` message. -(Notice that this configuration implies that directory nodes do not currently -talk to each other). - -The syntax of `handshake` is: -json serialized: - {"app-name": "joinmarket", - "directory": false, - "location-string": "hex-key@host:port", - "proto-ver": 5, - "features": {}, - "nick": "J5***" - } -Note that `proto-ver` is the version specified as `JM_VERSION` in jmdaemon.protocol. -(It has not changed for many years, it only specifies the syntax of the messages). -The `features` field is currently empty. - -The syntax of `dn-handshake` is: -json serialized: - {"app-name": "joinmarket", - "directory": true, - "proto-ver-min": 5, - "proto-ver-max": 5, - "features": {} - "accepted": true, - "nick": "J5**" - } - - Non-directory nodes should send `handshake` to directory nodes, - and directory nodes should return the `dn-handshake` method with `true` - for accepted, if and only if: - * the protocol version is in the accepted range - * the `directory` field of the peer is false - * the `app-name` is joinmarket - * the set of features requested is both recognized and accepted (currently: none) - * the nick used by this entity/bot across all message channels, used for cross-channel message spoofing protection - -Notice that more than one nick is NOT allowed per LN node; this is deferred to -future updates. - - In case those conditions are met, return `"accepted": true`, else return - `"accepted": false` and immediately disconnect the new peer. - (in this rejection case, the remaining fields of the `dn-handshake` message do - not matter, but can be kept as before for convenience). - -In case of a direct connection between peers (neither are directory nodes), -the party which connects then sends the first `handshake` message, and the -connected-to party responds with their own `handshake`. - -In this case, the connection should be accepted and maintained by the receiver -if and only if: -* the protocol version is identical -* the `directory` field of the peer is false -* the `app-name` is joinmarket -* the set of features is both recognized and accepted (currently: none) - -otherwise the peer should be immediately disconnected. - -ALL OTHER MESSAGES (control or otherwise, as detailed below), cannot be sent/ -will be ignored until the above two-way handshake is complete. - -#### OTHER CONTROL MESSAGES - -The syntax of `peerlist` is: - -nick || NICK_PEERLOCATOR_SEPARATOR || peer-location || "," ... (repeated) - -i.e. a serialized list of two-element tuples, each of which is a Joinmarket nick -followed by a peer location. - -`peerlist` may be sent by directory nodes to non-directory nodes at any time, -but currently it is sent according to a specific rule described below. - -#### LOCAL CONTROL MESSAGES - -There are two messages passed inside the plugin, to Joinmarketd, in response to events in lightningd, -namely the `connect` and `disconnect` events triggered at the LN level. These are used to update -the *state* of existing peers that we have recorded as connected at some point, to ourselves. - -The mechanisms here are loosely synchronizing a database of JM peers, with obviously the directory -node(s) acting as the data provider. It's notable that there are no guarantees of accuracy or -synchrony here. - -### PARSING OF RECEIVED JM_MESSAGES - - -The text will be utf-encoded before being hexlified, and therefore the following actions are needed of the receiver: - -Extract the peerid of the sender, and the actual message, received as encoded json: - -* peerid: json.loads(msg.decode("utf-8"))["peer_id"] -* message: json.loads(msg.decode("utf-8"))["payload"] -(which is prepended by a two byte message_type; see below). - -##### peerid: - -This field comes in three potential forms (as hex): -"00" : special null peerid indicating a local control message (see above). -"hex-key": peerid without connection information, allowing us to record the existence of a peer, - but not to send messages to it directly (only via the directory node). -"hex-key@host:port": peerid with connection information, allowing us to attempt to connect to it, - and send private messages to it directly. - -##### payload: - -This is parsed by: - -1. Take the first two hex-encoded bytes and convert to an integer: this is the - message type. Take the remaining part of the hex string and unhexlify, - converting to binary, then .decode("utf-8") again, converting to a string. - This means encoding is sometimes very inefficient, if the underlying string actually - contains hex or other encoding, but can't really be changed until the underlying - Joinmarket messaging protocol is changed. - -2. Split the decoded string by COMMAND PREFIX and parse as `from nick, to nick, command, message(s)` - (see above for syntax). - -The resulting messages can be passed into Joinmarket's normal message channel processing, and should be -identical to that coming from IRC. - -However, before doing so, we need to identify "public messages" versus private, which does not -have as natural a meaning here as it does on IRC; we impose it by using a to-nick value of PUBLIC -and by sending the message_type `687` to the Lightning RPC instead of the default -message_type `685` for privmsgs to a single counterparty. This will instruct the directory node -to send the message to every peer it knows about. - -### GETTING INFORMATION ABOUT PEERS FOR DIRECT CONNECTIONS - -To avoid passing huge lists of peers around, the directory node takes a "lazy" approach to -sharing connection info between peers: - -When Peer J51 asks to privmsg J52 (which it discovered when receiving a privmsg from J52, usually -here that would be in response to a `!orderbook` pubmsg by J51), the directory node does as instructed, -but then sends also a `peerlist` message to J51, containing the full network location of J52. - -Given this new information, J51 opportunistically tries to connect to J52 directly and if the network -connection succeeds, sends a handshake to J52. If J52 responds with acceptance, the direct messaging -connection is established, and from then on, until J51 sees a disconnect event for that network peer, -he will divert any `privmsg` to that party to use the direct connection instead of the directory node. - -""" """ this passthrough protocol allows the joinmarket daemon to receive messages @@ -826,6 +653,8 @@ def forward_privmsg_to_peer(self, nick, message, from_nick): assert self.self_as_peer.directory peerid = self.get_peerid_by_nick(nick) if not peerid: + log.debug("We were asked to send a message from {} to {}, " + "but {} is not connected.".format(from_nick, nick, nick)) return # The `message` passed in has format COMMAND_PREFIX||command||" "||msg # we need to parse out cmd, message for sending. diff --git a/test/e2e-coinjoin-test.py b/test/e2e-coinjoin-test.py index 496cd6e71..f55e06778 100644 --- a/test/e2e-coinjoin-test.py +++ b/test/e2e-coinjoin-test.py @@ -21,7 +21,9 @@ import random import json from datetime import datetime -from jmbase import get_nontor_agent, BytesProducer, jmprint, get_log, stop_reactor +from jmbase import (get_nontor_agent, BytesProducer, jmprint, + get_log, stop_reactor, hextobin, bintohex) +from jmbitcoin import privkey_to_pubkey from jmclient import (YieldGeneratorBasic, load_test_config, jm_single, JMClientProtocolFactory, start_reactor, SegwitWallet, get_mchannels, SegwitLegacyWallet, SNICKERClientProtocolFactory, SNICKERReceiver, @@ -42,6 +44,10 @@ wallet_name = "test-ln-yg-runner.jmdat" +mean_amt = 2.0 + +directory_node_indices = [1, 2] + # Note for tests of Lightning message channels: # (this data is not really needed *here* as the bots # retrieve their keys on startup with RPC `getinfo`, but is here @@ -50,7 +56,7 @@ # These nodes are generated with private keys (using --dev-force-privkey): # (note of course that c-lightning must be built with --enable-developer) # 121212121212121212121212121212121212121212121212121212121212121$i -# with $i 1..3 +# with $i 1..3 (or n in general, see 'regtest-count' below. # # the passthrough ports are 4910$i and they all run on localhost (this is default for regtest). # the lightning node ports are 9735+$i @@ -86,16 +92,28 @@ # only way in which the daemon processing is dependent on the config (the other elements # all occur during the aforementioned setup). # -def get_ln_messaging_config_regtest(run_num: int): +def get_ln_messaging_config_regtest(run_num: int, dns=[1]): """ Sets a ln messaging channel section for a regtest instance - indexed by `run_num`. + indexed by `run_num`. The indices to be used as directory nodes + should be passed as `dns`, as a list of ints. """ rpc_location = os.path.join(jm_single().datadir, "lightning-regtest"+str(run_num), "regtest", "lightning-rpc") passthrough_port = 49100 + run_num - # This node corresponds to run_num=1: - dn_nodes_list = "03df15dbd9e20c811cc5f4155745e89540a0b83f33978317cebe9dfc46c5253c55@127.0.0.1:9736" + def location_string(directory_node_run_num): + return bintohex(privkey_to_pubkey( + hextobin("12"*31 + "1" + str( + directory_node_run_num) + "01"))) + "@127.0.0.1:" + str( + 9735 + directory_node_run_num) + if run_num in dns: + # means *we* are a dn, and dns currently + # do not use other dns: + dns_to_use = [location_string(run_num)] + else: + dns_to_use = [location_string(a) for a in dns] + dn_nodes_list = ",".join(dns_to_use) + log.info("For node: {}, set dn list to: {}".format(run_num, dn_nodes_list)) return {"type": "ln-onion", "lightning-rpc": rpc_location, "passthrough-port": passthrough_port, @@ -103,6 +121,11 @@ def get_ln_messaging_config_regtest(run_num: int): class RegtestJMClientProtocolFactory(JMClientProtocolFactory): i = 1 + def set_directory_nodes(self, dns): + # a list of integers representing the directory nodes + # for this test: + self.dns = dns + def get_mchannels(self): # swaps out any existing lightning configs # in the config settings on startup, for one @@ -116,7 +139,7 @@ def get_mchannels(self): continue new_chans.append(c) if ln_found: - new_chans.append(get_ln_messaging_config_regtest(self.i)) + new_chans.append(get_ln_messaging_config_regtest(self.i, self.dns)) return new_chans class JMWalletDaemonT(JMWalletDaemon): @@ -183,21 +206,11 @@ def response_handler(self, response, handler): yield handler(body) return True -@pytest.mark.parametrize( - "num_ygs, wallet_structures, fb_indices, mean_amt", - [ - # 1sp 2yg, honest makers, one maker has FB: - (3, [[1, 3, 0, 0, 0]] * 4, [], 2), - # 1sp 3yg - #(3, [[1, 3, 0, 0, 0]] * 4, [], 2), - # 1 sp 9 ygs - #(9, [[1, 3, 0, 0, 0]] * 10, [], 2), - ]) -def test_start_ygs(setup_ln_ygrunner, num_ygs, wallet_structures, fb_indices, - mean_amt): - """Set up some wallets, for the ygs and 1 sp. - Then start the ygs in background and publish - the seed of the sp wallet for easy import into -qt +def test_start_yg_and_taker_setup(setup_ln_ygrunner): + """Set up some wallets, for the ygs and 1 taker. + Then start LN and the ygs in the background, then fire + a startup of a wallet daemon for the taker who then + makes a coinjoin payment. """ if jm_single().config.get("POLICY", "native") == "true": walletclass = SegwitWallet @@ -205,20 +218,23 @@ def test_start_ygs(setup_ln_ygrunner, num_ygs, wallet_structures, fb_indices, # TODO add Legacy walletclass = SegwitLegacyWallet + start_bot_num, end_bot_num = [int(x) for x in jm_single().config.get( + "MESSAGING:lightning1", "regtest-count").split(",")] + num_ygs = end_bot_num - start_bot_num + # specify the number of wallets and bots of each type: wallet_services = make_wallets(num_ygs + 1, - wallet_structures=wallet_structures, - mean_amt=mean_amt, - walletclass=walletclass, - fb_indices=fb_indices) + wallet_structures=[[1, 3, 0, 0, 0]] * (num_ygs + 1), + mean_amt=2.0, + walletclass=walletclass) #the sendpayment bot uses the last wallet in the list - wallet_service = wallet_services[num_ygs]['wallet'] - jmprint("\n\nTaker wallet seed : " + wallet_services[num_ygs]['seed']) + wallet_service = wallet_services[end_bot_num - 1]['wallet'] + jmprint("\n\nTaker wallet seed : " + wallet_services[end_bot_num - 1]['seed']) # for manual audit if necessary, show the maker's wallet seeds # also (note this audit should be automated in future, see # test_full_coinjoin.py in this directory) jmprint("\n\nMaker wallet seeds: ") - for i in range(num_ygs): - jmprint("Maker seed: " + wallet_services[i]['seed']) + for i in range(start_bot_num, end_bot_num): + jmprint("Maker seed: " + wallet_services[i - 1]['seed']) jmprint("\n") wallet_service.sync_wallet(fast=True) ygclass = YieldGeneratorBasic @@ -259,38 +275,27 @@ def test_start_ygs(setup_ln_ygrunner, num_ygs, wallet_structures, fb_indices, ordertype = prefix + ordertype - for i in range(num_ygs): + for i in range(start_bot_num, end_bot_num): cfg = [txfee_contribution, cjfee_a, cjfee_r, ordertype, minsize, txfee_contribution_factor, cjfee_factor, size_factor] - wallet_service_yg = wallet_services[i]["wallet"] + wallet_service_yg = wallet_services[i - 1]["wallet"] wallet_service_yg.startService() yg = ygclass(wallet_service_yg, cfg) - if i in fb_indices: - # create a timelocked address and fund it; - # must be done after sync, so deferred: - wallet_service_yg.timelock_funded = False - sync_wait_loop = task.LoopingCall(get_addr_and_fund, yg) - sync_wait_loop.start(1.0, now=False) clientfactory = RegtestJMClientProtocolFactory(yg, proto_type="MAKER") # This ensures that the right rpc/port config is passed into the daemon, # for this specific bot: - clientfactory.i = i + 1 - if jm_single().config.get("SNICKER", "enabled") == "true": - snicker_r = SNICKERReceiver(wallet_service_yg) - servers = jm_single().config.get("SNICKER", "servers").split(",") - snicker_factory = SNICKERClientProtocolFactory(snicker_r, servers) - else: - snicker_factory = None + clientfactory.i = i + # This ensures that this bot knows which other bots are directory nodes: + clientfactory.set_directory_nodes(directory_node_indices) nodaemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if nodaemon == 1 else False #rs = True if i == num_ygs - 1 else False start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), - clientfactory, snickerfactory=snicker_factory, - daemon=daemon, rs=False) - reactor.callLater(1.0, start_test_taker, wallet_services[num_ygs]['wallet'], 4) + clientfactory, daemon=daemon, rs=False) + reactor.callLater(1.0, start_test_taker, wallet_services[end_bot_num - 1]['wallet'], 4) reactor.run() @defer.inlineCallbacks @@ -316,6 +321,7 @@ def get_client_factory(): clientfactory = RegtestJMClientProtocolFactory(mgr.daemon.taker, proto_type="TAKER") clientfactory.i = i + clientfactory.set_directory_nodes(directory_node_indices) return clientfactory mgr.daemon.get_client_factory = get_client_factory diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 54090c961..37323c62e 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -46,11 +46,8 @@ clightning-location = bundled directory-nodes = 03df15dbd9e20c811cc5f4155745e89540a0b83f33978317cebe9dfc46c5253c55@localhost:9735 passthrough-port = 49101 lightning-port = 9736 -# means we use indices 1,2,3,4: -regtest-count=1,4 - -# Note: for now, lightning messaging section is created -# on the fly for the ln-onion specific test. +# means we use indices 1,2,3,4,5: +regtest-count=1,5 [TIMEOUT] maker_timeout_sec = 10