From 075b416f560463c818f8cf2c5821c5a9dd7c2aca Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 2 Jun 2016 11:52:24 -0400 Subject: [PATCH 01/53] --- bitcore start --- From 9babc7ff9fa19b93cf7b3ef2f73ec4e66f8c132f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 5 Mar 2016 16:31:10 -0500 Subject: [PATCH 02/53] main: start of address index Adds a configuration option for addressindex to search for txids by address. Includes an additional rpc method for getting the txids for an address. --- src/init.cpp | 2 ++ src/main.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++ src/main.h | 38 ++++++++++++++++++++++++++ src/rpcmisc.cpp | 36 +++++++++++++++++++++++++ src/rpcserver.cpp | 3 +++ src/rpcserver.h | 2 ++ src/script/script.cpp | 11 ++++++++ src/script/script.h | 2 ++ src/txdb.cpp | 33 +++++++++++++++++++++++ src/txdb.h | 3 +++ 10 files changed, 192 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 0b3234566a9b6..16598ada44e39 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -350,6 +350,8 @@ std::string HelpMessage(HelpMessageMode mode) #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); + strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); diff --git a/src/main.cpp b/src/main.cpp index f85ac331770ba..adad65640ba06 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -66,6 +66,7 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fAddressIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1438,6 +1439,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +{ + if (!fAddressIndex) + return error("%s: address index not enabled"); + + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) + return error("%s: unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2322,9 +2334,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin std::vector > vPos; vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); + std::vector > addressIndex; + for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); nInputs += tx.vin.size(); nSigOps += GetLegacySigOpCount(tx); @@ -2351,6 +2366,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-txns-nonfinal"); } + if (fAddressIndex) + { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + } else { + continue; + } + } + } + if (fStrictPayToScriptHash) { // Add in sigops done by pay-to-script-hash inputs; @@ -2372,6 +2403,24 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin control.Add(vChecks); } + if (fAddressIndex) { + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + } else { + continue; + } + + } + } + + CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); @@ -2422,6 +2471,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); + if (fAddressIndex) + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3813,6 +3866,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); + // Check whether we have an address index + pblocktree->ReadFlag("addressindex", fAddressIndex); + LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3973,6 +4030,11 @@ bool InitBlockIndex(const CChainParams& chainparams) // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", DEFAULT_TXINDEX); pblocktree->WriteFlag("txindex", fTxIndex); + + // Use the provided setting for -addressindex in the new database + fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); + pblocktree->WriteFlag("addressindex", fAddressIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 1a696dcd91ff8..159099036dcf4 100644 --- a/src/main.h +++ b/src/main.h @@ -111,6 +111,7 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static const bool DEFAULT_ADDRESSINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -290,6 +291,42 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CAddressIndexKey { + uint160 hashBytes; + unsigned int type; + uint256 txhash; + size_t index; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(hashBytes); + READWRITE(type); + READWRITE(txhash); + READWRITE(index); + } + + CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { + hashBytes = addressHash; + type = addressType; + txhash = txid; + index = txindex; + } + + CAddressIndexKey() { + SetNull(); + } + + void SetNull() { + hashBytes.SetNull(); + type = 0; + txhash.SetNull(); + index = 0; + } + +}; + struct CDiskTxPos : public CDiskBlockPos { unsigned int nTxOffset; // after header @@ -420,6 +457,7 @@ class CScriptCheck ScriptError GetScriptError() const { return error; } }; +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9871c3fcc9031..aa762af88f8be 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -396,3 +396,39 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + CBitcoinAddress address(params[0].get_str()); + if (!address.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + CKeyID keyID; + address.GetKeyID(keyID); + + int type = 1; // TODO + std::vector > addressIndex; + + LOCK(cs_main); + + if (!GetAddressIndex(keyID, type, addressIndex)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) + result.push_back(it->first.txhash.GetHex()); + + return result; + +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index b3abeec4a3ce8..67da51ceb75af 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -313,6 +313,9 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false }, #endif + /* Address index */ + { "addressindex", "getaddresstxids", &getaddresstxids, false }, + /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ diff --git a/src/rpcserver.h b/src/rpcserver.h index babf7c8d2e1f2..11f7600511a9c 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,8 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddresstxids(const UniValue& params, bool fHelp); + extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); extern UniValue addnode(const UniValue& params, bool fHelp); diff --git a/src/script/script.cpp b/src/script/script.cpp index 9f2809e593756..3e4b72f5d9652 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -201,6 +201,17 @@ unsigned int CScript::GetSigOpCount(const CScript& scriptSig) const return subscript.GetSigOpCount(true); } +bool CScript::IsPayToPublicKeyHash() const +{ + // Extra-fast test for pay-to-pubkey-hash CScripts: + return (this->size() == 25 && + (*this)[0] == OP_DUP && + (*this)[1] == OP_HASH160 && + (*this)[2] == 0x14 && + (*this)[23] == OP_EQUALVERIFY && + (*this)[24] == OP_CHECKSIG); +} + bool CScript::IsPayToScriptHash() const { // Extra-fast test for pay-to-script-hash CScripts: diff --git a/src/script/script.h b/src/script/script.h index d2a68a07ba149..9dd75a5583668 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -608,6 +608,8 @@ class CScript : public CScriptBase */ unsigned int GetSigOpCount(const CScript& scriptSig) const; + bool IsPayToPublicKeyHash() const; + bool IsPayToScriptHash() const; /** Called by IsStandardTx and P2SH/BIP62 VerifyScript (which makes it consensus-critical). */ diff --git a/src/txdb.cpp b/src/txdb.cpp index f99e11f26e3f4..153d7b84c9ad0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,6 +21,7 @@ using namespace std; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; +static const char DB_ADDRESSINDEX = 'a'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -163,6 +164,38 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + CAmount nValue; + if (pcursor->GetValue(nValue)) { + addressIndex.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address index value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index 22e0c5704cb26..3d2ace581a018 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressIndexKey; class uint256; //! -dbcache default (MiB) @@ -57,6 +58,8 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool WriteAddressIndex(const std::vector > &vect); + bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From 73b2d0851b5256c8abfe92763d96b60662bd5ed8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 14:47:12 -0500 Subject: [PATCH 03/53] test: added unit tests for CScript.IsPayToPublicKeyHash --- src/Makefile.test.include | 1 + src/test/script_P2PKH_tests.cpp | 59 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/test/script_P2PKH_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e96e7bec37d5f..4ea09116c2d1b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -70,6 +70,7 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ + test/script_P2PKH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ diff --git a/src/test/script_P2PKH_tests.cpp b/src/test/script_P2PKH_tests.cpp new file mode 100644 index 0000000000000..3a7dc16608aaa --- /dev/null +++ b/src/test/script_P2PKH_tests.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2012-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "script/script.h" +#include "test/test_bitcoin.h" + +#include + +using namespace std; + +BOOST_FIXTURE_TEST_SUITE(script_P2PKH_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(IsPayToPublicKeyHash) +{ + // Test CScript::IsPayToPublicKeyHash() + uint160 dummy; + CScript p2pkh; + p2pkh << OP_DUP << OP_HASH160 << ToByteVector(dummy) << OP_EQUALVERIFY << OP_CHECKSIG; + BOOST_CHECK(p2pkh.IsPayToPublicKeyHash()); + + static const unsigned char direct[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(CScript(direct, direct+sizeof(direct)).IsPayToPublicKeyHash()); + + static const unsigned char notp2pkh1[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(notp2pkh1, notp2pkh1+sizeof(notp2pkh1)).IsPayToPublicKeyHash()); + + static const unsigned char p2sh[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL + }; + BOOST_CHECK(!CScript(p2sh, p2sh+sizeof(p2sh)).IsPayToPublicKeyHash()); + + static const unsigned char extra[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(extra, extra+sizeof(extra)).IsPayToPublicKeyHash()); + + static const unsigned char missing[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_RETURN + }; + BOOST_CHECK(!CScript(missing, missing+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char missing2[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + }; + BOOST_CHECK(!CScript(missing2, missing2+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char tooshort[] = { + OP_DUP, OP_HASH160, 2, 0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(tooshort, tooshort+sizeof(direct)).IsPayToPublicKeyHash()); + +} + +BOOST_AUTO_TEST_SUITE_END() From 4d461956349551c67567791bcb18e58a22df96b6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 16:15:49 -0500 Subject: [PATCH 04/53] qa: started test for addressindex rpc getaddresstxids --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/addressindex.py | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 qa/rpc-tests/addressindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 37014cf76acdd..804914e29ea5b 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -97,6 +97,7 @@ 'walletbackup.py', 'nodehandling.py', 'reindex.py', + 'addressindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py new file mode 100755 index 0000000000000..c4353d92a6f30 --- /dev/null +++ b/qa/rpc-tests/addressindex.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class AddressIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + assert_equal(self.nodes[1].getbalance(), 0) + assert_equal(self.nodes[2].getbalance(), 0) + + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + self.sync_all() + + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); + assert_equal(len(txids), 3); + +if __name__ == '__main__': + AddressIndexTest().main() From 18ea599a71614faf78d973df42ec0468a5bd86f3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 11:27:30 -0500 Subject: [PATCH 05/53] main: index address index sorted by height --- qa/rpc-tests/addressindex.py | 11 ++++++-- src/main.cpp | 9 +++--- src/main.h | 54 ++++++++++++++++++++++++++++++------ src/txdb.cpp | 4 +-- src/txdb.h | 1 + 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c4353d92a6f30..f3cbc2ef848e9 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -41,14 +41,19 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) - txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) - txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) - txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) self.nodes[0].generate(1) self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); assert_equal(len(txids), 3); + assert_equal(txids[0], txid0); + assert_equal(txids[1], txid1); + assert_equal(txids[2], txid2); if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index adad65640ba06..0ad52abbb6097 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2372,10 +2372,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else { continue; } @@ -2409,10 +2409,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else { continue; } @@ -2420,7 +2420,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } - CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); diff --git a/src/main.h b/src/main.h index 159099036dcf4..c2196fcbd741f 100644 --- a/src/main.h +++ b/src/main.h @@ -292,26 +292,33 @@ struct CNodeStateStats { }; struct CAddressIndexKey { - uint160 hashBytes; unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; uint256 txhash; - size_t index; + size_t outindex; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(hashBytes); READWRITE(type); + READWRITE(hashBytes); + READWRITE(blockHeight); + READWRITE(txindex); READWRITE(txhash); - READWRITE(index); + READWRITE(outindex); } - CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { - hashBytes = addressHash; + CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t outputIndex) { type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; txhash = txid; - index = txindex; + outindex = outputIndex; } CAddressIndexKey() { @@ -319,12 +326,41 @@ struct CAddressIndexKey { } void SetNull() { - hashBytes.SetNull(); type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; txhash.SetNull(); - index = 0; + outindex = 0; + } + +}; + +struct CAddressIndexIteratorKey { + unsigned int type; + uint160 hashBytes; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(type); + READWRITE(hashBytes); + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { + type = addressType; + hashBytes = addressHash; + } + + CAddressIndexIteratorKey() { + SetNull(); } + void SetNull() { + type = 0; + hashBytes.SetNull(); + } }; struct CDiskTxPos : public CDiskBlockPos diff --git a/src/txdb.cpp b/src/txdb.cpp index 153d7b84c9ad0..e93594b9ed80f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -167,7 +167,7 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) - batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); return WriteBatch(batch); } @@ -175,7 +175,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); while (pcursor->Valid()) { boost::this_thread::interruption_point(); diff --git a/src/txdb.h b/src/txdb.h index 3d2ace581a018..ca32869ccc37d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -18,6 +18,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; +struct CAddressIndexIteratorKey; class uint256; //! -dbcache default (MiB) From fcac6bcdc82008376fae530a07ae002e64b44c40 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 17:40:40 -0500 Subject: [PATCH 06/53] rpc: fix issue for querying txids for p2sh addresses --- qa/rpc-tests/addressindex.py | 13 +++++++++++++ src/base58.cpp | 17 +++++++++++++++++ src/base58.h | 1 + src/rpcmisc.cpp | 11 +++++------ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index f3cbc2ef848e9..88533d028fc5a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -42,11 +42,17 @@ def run_test(self): assert_equal(self.nodes[2].getbalance(), 0) txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) + self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); @@ -55,5 +61,12 @@ def run_test(self): assert_equal(txids[1], txid1); assert_equal(txids[2], txid2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsb), 3); + assert_equal(txidsb[0], txidb0); + assert_equal(txidsb[1], txidb1); + assert_equal(txidsb[2], txidb2); + + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/base58.cpp b/src/base58.cpp index 5e26cf8d4738f..80fa99c2ab082 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -262,6 +262,23 @@ CTxDestination CBitcoinAddress::Get() const return CNoDestination(); } +bool CBitcoinAddress::GetIndexKey(uint160& hashBytes, int& type) const +{ + if (!IsValid()) { + return false; + } else if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 1; + return true; + } else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 2; + return true; + } + + return false; +} + bool CBitcoinAddress::GetKeyID(CKeyID& keyID) const { if (!IsValid() || vchVersion != Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) diff --git a/src/base58.h b/src/base58.h index a3980118aae34..b8280b1dd621f 100644 --- a/src/base58.h +++ b/src/base58.h @@ -116,6 +116,7 @@ class CBitcoinAddress : public CBase58Data { CTxDestination Get() const; bool GetKeyID(CKeyID &keyID) const; + bool GetIndexKey(uint160& hashBytes, int& type) const; bool IsScript() const; }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index aa762af88f8be..0a39d2fbcb3da 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -411,18 +411,17 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) ); CBitcoinAddress address(params[0].get_str()); - if (!address.IsValid()) + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } - CKeyID keyID; - address.GetKeyID(keyID); - - int type = 1; // TODO std::vector > addressIndex; LOCK(cs_main); - if (!GetAddressIndex(keyID, type, addressIndex)) + if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); UniValue result(UniValue::VARR); From 2500d1d1159b2021e2bdcdde3c1e7d4f04917d90 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 19:45:08 -0500 Subject: [PATCH 07/53] rpc: update getaddresstxids for uniqueness --- qa/rpc-tests/addressindex.py | 28 ++++++++++++++++++++++++++++ src/rpcmisc.cpp | 10 ++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 88533d028fc5a..fd750403f7a11 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -9,6 +9,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii class AddressIndexTest(BitcoinTestFramework): @@ -41,6 +44,9 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check p2pkh and p2sh address indexes + print "Testing p2pkh and p2sh address index..." + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) @@ -67,6 +73,28 @@ def run_test(self): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that outputs with the same address will only return one txid + print "Testing for txid uniqueness..." + addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") + scriptPubKey = CScript([OP_HASH160, addressHash, OP_EQUAL]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + + self.nodes[0].generate(1) + self.sync_all() + + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsmany), 4); + assert_equal(txidsmany[3], sent_txid); + + print "Passed\n" + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 0a39d2fbcb3da..cca8a9896ab2d 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -424,9 +424,15 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + std::set txids; + UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) - result.push_back(it->first.txhash.GetHex()); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string txid = it->first.txhash.GetHex(); + if (txids.insert(txid).second) { + result.push_back(txid); + } + } return result; From f4d11ffc7c3fd875ea8d464e25c7c23494c6e7e9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 15:25:01 -0400 Subject: [PATCH 08/53] rpc: query for multiple addresses txids --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 46 +++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index fd750403f7a11..a985735f88fd3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -73,6 +73,10 @@ def run_test(self): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that multiple addresses works + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); + assert_equal(len(multitxids), 6); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index cca8a9896ab2d..a7bf7aad2330c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -410,20 +410,50 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) "]\n" ); - CBitcoinAddress address(params[0].get_str()); - uint160 hashBytes; - int type = 0; - if (!address.GetIndexKey(hashBytes, type)) { + std::vector > addresses; + + if (params[0].isStr()) { + + CBitcoinAddress address(params[0].get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + + } else if (params[0].isObject()) { + + UniValue addressValues = find_value(params[0].get_obj(), "addresses"); + if (!addressValues.isArray()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Addresses is expected to be an array"); + } + + std::vector values = addressValues.getValues(); + + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + + CBitcoinAddress address(it->get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + } + } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } std::vector > addressIndex; - LOCK(cs_main); - - if (!GetAddressIndex(hashBytes, type, addressIndex)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + // TODO sort by height std::set txids; UniValue result(UniValue::VARR); From 5b5f3f7d00465200315a06cf40f4ce31a01d0d50 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 16:24:55 -0400 Subject: [PATCH 09/53] rpc: sort txids by height for multiple addresses --- qa/rpc-tests/addressindex.py | 12 ++++++++++++ src/rpcmisc.cpp | 13 ++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a985735f88fd3..2e15348d6be1b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -48,14 +48,20 @@ def run_test(self): print "Testing p2pkh and p2sh address index..." txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) @@ -76,6 +82,12 @@ def run_test(self): # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); assert_equal(len(multitxids), 6); + assert_equal(multitxids[0], txid0); + assert_equal(multitxids[1], txidb0); + assert_equal(multitxids[2], txid1); + assert_equal(multitxids[3], txidb1); + assert_equal(multitxids[4], txid2); + assert_equal(multitxids[5], txidb2); # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index a7bf7aad2330c..e04b77d758c8c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -453,17 +453,24 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - // TODO sort by height std::set txids; + std::vector > vtxids; - UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); if (txids.insert(txid).second) { - result.push_back(txid); + vtxids.push_back(std::make_pair(height, txid)); } } + std::sort(vtxids.begin(), vtxids.end()); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { + result.push_back(it->second); + } + return result; } From 7959a190850db293fa3faf6251e270738581a764 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:00:50 -0400 Subject: [PATCH 10/53] main: serialize height in BE for address index key fixes a sorting issue when iterating over keys --- src/main.h | 49 +++++++++++++++++++++++++++++++++---------------- src/serialize.h | 11 +++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/main.h b/src/main.h index c2196fcbd741f..13791525333de 100644 --- a/src/main.h +++ b/src/main.h @@ -299,16 +299,27 @@ struct CAddressIndexKey { uint256 txhash; size_t outindex; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); - READWRITE(blockHeight); - READWRITE(txindex); - READWRITE(txhash); - READWRITE(outindex); + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, outindex); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + outindex = ser_readdata32(s); } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, @@ -340,12 +351,18 @@ struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); + size_t GetSerializeSize(int nType, int nVersion) const { + return 21; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); } CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { diff --git a/src/serialize.h b/src/serialize.h index 5c2db9d332ce3..41a51d237440c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -91,6 +91,11 @@ template inline void ser_writedata32(Stream &s, uint32_t obj) obj = htole32(obj); s.write((char*)&obj, 4); } +template inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} template inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); @@ -114,6 +119,12 @@ template inline uint32_t ser_readdata32(Stream &s) s.read((char*)&obj, 4); return le32toh(obj); } +template inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} template inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; From 7dbbb79cecb5f9f06c9406eaa680e047f7024af6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:50:19 -0400 Subject: [PATCH 11/53] rpc: only sort when combining multiple results It's only necessary to sort when combining results for several addresses as the results are already in order from the database. --- src/rpcmisc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index e04b77d758c8c..1a86353e27380 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -464,7 +464,9 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::sort(vtxids.begin(), vtxids.end()); + if (addresses.size() > 1) { + std::sort(vtxids.begin(), vtxids.end()); + } UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { From 5bb6d69ff849732f5695e221a06d50547a08db0a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:06:08 -0400 Subject: [PATCH 12/53] rpc: added getaddressbalance method using addressindex --- qa/rpc-tests/addressindex.py | 12 ++++++ src/rpcmisc.cpp | 83 ++++++++++++++++++++++++++++-------- src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 2e15348d6be1b..8f123eaa075f6 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -44,6 +44,10 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 0); + # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -89,6 +93,10 @@ def run_test(self): assert_equal(multitxids[4], txid2); assert_equal(multitxids[5], txidb2); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") @@ -109,6 +117,10 @@ def run_test(self): assert_equal(len(txidsmany), 4); assert_equal(txidsmany[3], sent_txid); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000 + 21); + print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1a86353e27380..d4ad29e65581c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -397,23 +397,9 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } -UniValue getaddresstxids(const UniValue& params, bool fHelp) +bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { - if (fHelp || params.size() != 1) - throw runtime_error( - "getaddresstxids\n" - "\nReturns the txids for an address (requires addressindex to be enabled).\n" - "\nResult\n" - "[\n" - " \"transactionid\" (string) The transaction id\n" - " ,...\n" - "]\n" - ); - - std::vector > addresses; - if (params[0].isStr()) { - CBitcoinAddress address(params[0].get_str()); uint160 hashBytes; int type = 0; @@ -421,7 +407,6 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } addresses.push_back(std::make_pair(hashBytes, type)); - } else if (params[0].isObject()) { UniValue addressValues = find_value(params[0].get_obj(), "addresses"); @@ -445,9 +430,73 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + return true; + + +} + +UniValue getaddressbalance(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressbalance\n" + "\nReturns the balance for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "{\n" + " \"balance\" (string) The current balance\n" + " ,...\n" + "}\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + CAmount balance = 0; + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + balance += it->second; + } + + UniValue result(UniValue::VOBJ); + result.push_back(Pair("balance", balance)); + + return result; + +} + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + std::vector > addressIndex; - for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 67da51ceb75af..da66b1c0a1a97 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = /* Address index */ { "addressindex", "getaddresstxids", &getaddresstxids, false }, + { "addressindex", "getaddressbalance", &getaddressbalance, false }, /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 11f7600511a9c..8a80fc5ee2abe 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -167,6 +167,7 @@ extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp extern UniValue getaddresstxids(const UniValue& params, bool fHelp); +extern UniValue getaddressbalance(const UniValue& params, bool fHelp); extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); From 2e8a4c00fa03f8efe2b59ef9f32e9683f1f6472e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:40:16 -0400 Subject: [PATCH 13/53] rpc: add receieved to balance --- src/rpcmisc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index d4ad29e65581c..549e6c7a5b0f5 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -463,13 +463,18 @@ UniValue getaddressbalance(const UniValue& params, bool fHelp) } CAmount balance = 0; + CAmount received = 0; for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + if (it->second > 0) { + received += it->second; + } balance += it->second; } UniValue result(UniValue::VOBJ); result.push_back(Pair("balance", balance)); + result.push_back(Pair("received", received)); return result; From 24deb4efc146848269c4fecb2ae6db62930307f2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 13:55:19 -0400 Subject: [PATCH 14/53] rpc: include height in getrawtransaction results --- src/rpcrawtransaction.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index bd51aa0ab018d..6ab1807d46373 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -101,12 +101,14 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (mi != mapBlockIndex.end() && (*mi).second) { CBlockIndex* pindex = (*mi).second; if (chainActive.Contains(pindex)) { + entry.push_back(Pair("height", pindex->nHeight)); entry.push_back(Pair("confirmations", 1 + chainActive.Height() - pindex->nHeight)); entry.push_back(Pair("time", pindex->GetBlockTime())); entry.push_back(Pair("blocktime", pindex->GetBlockTime())); - } - else + } else { + entry.push_back(Pair("height", -1)); entry.push_back(Pair("confirmations", 0)); + } } } } From 935ca8f832cd3b7bddf762ce236225161ea6f1aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 18:11:04 -0400 Subject: [PATCH 15/53] main: add block timestamp index --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/timestampindex.py | 51 +++++++++++++++++++++++++++ src/main.cpp | 28 +++++++++++++-- src/main.h | 63 ++++++++++++++++++++++++++++++++++ src/rpcblockchain.cpp | 32 +++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 27 +++++++++++++++ src/txdb.h | 4 +++ 9 files changed, 206 insertions(+), 2 deletions(-) create mode 100755 qa/rpc-tests/timestampindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 804914e29ea5b..73bfe1e28b23b 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -98,6 +98,7 @@ 'nodehandling.py', 'reindex.py', 'addressindex.py', + 'timestampindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py new file mode 100755 index 0000000000000..46ff710bdf37f --- /dev/null +++ b/qa/rpc-tests/timestampindex.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test timestampindex generation and fetching +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TimestampIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-timestampindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-timestampindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining 5 blocks..." + blockhashes = self.nodes[0].generate(5) + low = self.nodes[0].getblock(blockhashes[0])["time"] + high = self.nodes[0].getblock(blockhashes[4])["time"] + self.sync_all() + print "Checking timestamp index..." + hashes = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes), 5) + assert_equal(sorted(blockhashes), sorted(hashes)) + print "Passed\n" + + +if __name__ == '__main__': + TimestampIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index 0ad52abbb6097..de65db40ac1a1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -67,6 +67,7 @@ bool fImporting = false; bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; +bool fTimestampIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1439,6 +1440,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +{ + if (!fTimestampIndex) + return error("Timestamp index not enabled"); + + if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + return error("Unable to get hashes for timestamps"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { if (!fAddressIndex) @@ -2471,8 +2483,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return AbortNode(state, "Failed to write transaction index"); if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) - return AbortNode(state, "Failed to write address index"); + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + + if (fTimestampIndex) + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + return AbortNode(state, "Failed to write timestamp index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3869,6 +3885,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a timestamp index + pblocktree->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -4034,6 +4054,10 @@ bool InitBlockIndex(const CChainParams& chainparams) fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); + // Use the provided setting for -timestampindex in the new database + fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + pblocktree->WriteFlag("timestampindex", fTimestampIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 13791525333de..fe0a3dd10de43 100644 --- a/src/main.h +++ b/src/main.h @@ -112,6 +112,7 @@ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; +static const bool DEFAULT_TIMESTAMPINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -291,6 +292,67 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -510,6 +572,7 @@ class CScriptCheck ScriptError GetScriptError() const { return error; } }; +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index f0bcafafe9500..276e99400c916 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -275,6 +275,38 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockhashes(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "getblockhashes timestamp\n" + "\nReturns array of hashes of blocks within the timestamp range provided.\n" + "\nArguments:\n" + "1. high (numeric, required) The newer block timestamp\n" + "2. low (numeric, required) The older block timestamp\n" + "\nResult:\n" + "[" + " \"hash\" (string) The block hash\n" + "]" + "\nExamples:\n" + ); + + unsigned int high = params[0].get_int(); + unsigned int low = params[1].get_int(); + std::vector blockHashes; + + if (!GetTimestampIndex(high, low, blockHashes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + } + + UniValue result(UniValue::VARR); + for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + result.push_back(it->GetHex()); + } + + return result; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index da66b1c0a1a97..51235c184f4df 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -278,6 +278,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "getbestblockhash", &getbestblockhash, true }, { "blockchain", "getblockcount", &getblockcount, true }, { "blockchain", "getblock", &getblock, true }, + { "blockchain", "getblockhashes", &getblockhashes, true }, { "blockchain", "getblockhash", &getblockhash, true }, { "blockchain", "getblockheader", &getblockheader, true }, { "blockchain", "getchaintips", &getchaintips, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 8a80fc5ee2abe..0be81809340be 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -258,6 +258,7 @@ extern UniValue getdifficulty(const UniValue& params, bool fHelp); extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); +extern UniValue getblockhashes(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblock(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index e93594b9ed80f..3aa6860583971 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -196,6 +197,32 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector &hashes) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { + hashes.push_back(key.second.blockHash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index ca32869ccc37d..7fefcaafafefa 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -19,6 +19,8 @@ class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CTimestampIndexKey; +struct CTimestampIndexIteratorKey; class uint256; //! -dbcache default (MiB) @@ -61,6 +63,8 @@ class CBlockTreeDB : public CDBWrapper bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From f76c2585f079b3e2e9c504db343ccd3c3897b3aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:14:36 -0400 Subject: [PATCH 16/53] test: added to for balance after spending --- qa/rpc-tests/addressindex.py | 87 +++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 8f123eaa075f6..59837f7838d97 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -46,7 +46,7 @@ def run_test(self): # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 0); + assert_equal(balance0["balance"], 0) # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -71,31 +71,31 @@ def run_test(self): self.sync_all() - txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); - assert_equal(len(txids), 3); - assert_equal(txids[0], txid0); - assert_equal(txids[1], txid1); - assert_equal(txids[2], txid2); + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs") + assert_equal(len(txids), 3) + assert_equal(txids[0], txid0) + assert_equal(txids[1], txid1) + assert_equal(txids[2], txid2) - txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsb), 3); - assert_equal(txidsb[0], txidb0); - assert_equal(txidsb[1], txidb1); - assert_equal(txidsb[2], txidb2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsb), 3) + assert_equal(txidsb[0], txidb0) + assert_equal(txidsb[1], txidb1) + assert_equal(txidsb[2], txidb2) # Check that multiple addresses works - multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); - assert_equal(len(multitxids), 6); - assert_equal(multitxids[0], txid0); - assert_equal(multitxids[1], txidb0); - assert_equal(multitxids[2], txid1); - assert_equal(multitxids[3], txidb1); - assert_equal(multitxids[4], txid2); - assert_equal(multitxids[5], txidb2); + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) + assert_equal(len(multitxids), 6) + assert_equal(multitxids[0], txid0) + assert_equal(multitxids[1], txidb0) + assert_equal(multitxids[2], txid1) + assert_equal(multitxids[3], txidb1) + assert_equal(multitxids[4], txid2) + assert_equal(multitxids[5], txidb2) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000); + assert_equal(balance0["balance"], 45 * 100000000) # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." @@ -107,19 +107,54 @@ def run_test(self): tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] tx.rehash() - signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) - sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) self.nodes[0].generate(1) self.sync_all() - txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsmany), 4); - assert_equal(txidsmany[3], sent_txid); + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsmany), 4) + assert_equal(txidsmany[3], sent_txid) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000 + 21); + assert_equal(balance0["balance"], 45 * 100000000 + 21) + + # Check that balances are correct after spending + privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG]) + self.nodes[0].importprivkey(privkey2) + + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey2)] + tx.rehash() + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + spending_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + balance1 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance1["balance"], amount) + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] + send_amount = 1 * 100000000 + 12840 + change_amount = amount - send_amount - 10000 + tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + balance2 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance2["balance"], change_amount) print "Passed\n" From 206882cd4b7a93c08579134986814b76c49564e2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:41:08 -0400 Subject: [PATCH 17/53] main: fixed bug with overlapping address index keys There was a bug where the spending address index could have the same key as the receiving address index if the input and output indexes matched. This lead to the output always overwriting the input index leading to incorrect balances with missing spent amounts. This patch separates the two so that they have unique keys so balances will be correctly calculated. --- qa/rpc-tests/addressindex.py | 2 +- src/main.cpp | 8 ++++---- src/main.h | 9 ++++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 59837f7838d97..0ffbcffd77439 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -145,7 +145,7 @@ def run_test(self): tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] send_amount = 1 * 100000000 + 12840 change_amount = amount - send_amount - 10000 - tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.vout = [CTxOut(change_amount, scriptPubKey2), CTxOut(send_amount, scriptPubKey)] tx.rehash() signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) diff --git a/src/main.cpp b/src/main.cpp index de65db40ac1a1..3826bc1980407 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2384,10 +2384,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else { continue; } @@ -2421,10 +2421,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else { continue; } diff --git a/src/main.h b/src/main.h index fe0a3dd10de43..3378b24df5916 100644 --- a/src/main.h +++ b/src/main.h @@ -360,6 +360,7 @@ struct CAddressIndexKey { unsigned int txindex; uint256 txhash; size_t outindex; + bool spending; size_t GetSerializeSize(int nType, int nVersion) const { return 65; @@ -373,6 +374,8 @@ struct CAddressIndexKey { ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, outindex); + char f = spending; + ser_writedata8(s, f); } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -382,16 +385,19 @@ struct CAddressIndexKey { txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); outindex = ser_readdata32(s); + char f = ser_readdata8(s); + spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex) { + uint256 txid, size_t outputIndex, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; outindex = outputIndex; + spending = isSpending; } CAddressIndexKey() { @@ -405,6 +411,7 @@ struct CAddressIndexKey { txindex = 0; txhash.SetNull(); outindex = 0; + spending = false; } }; From cad092aebb03fcb7884c947c0ecac26474ba4931 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 24 Mar 2016 15:44:23 -0400 Subject: [PATCH 18/53] main: get address deltas between range of block heights --- qa/rpc-tests/addressindex.py | 11 +++++++ src/main.cpp | 9 +++--- src/main.h | 36 +++++++++++++++++----- src/rpcmisc.cpp | 60 ++++++++++++++++++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 13 ++++++-- src/txdb.h | 4 ++- 8 files changed, 120 insertions(+), 15 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 0ffbcffd77439..4fc4cc557e361 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -156,6 +156,17 @@ def run_test(self): balance2 = self.nodes[1].getaddressbalance(address2) assert_equal(balance2["balance"], change_amount) + # Check that deltas are returned correctly + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) + balance3 = 0; + for delta in deltas: + balance3 += delta["satoshis"] + assert_equal(balance3, change_amount) + + # Check that deltas can be returned from range of block heights + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) + assert_equal(len(deltas), 1); + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 3826bc1980407..4b7c95afd4ec5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1451,13 +1451,14 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, int start, int end) { if (!fAddressIndex) - return error("%s: address index not enabled"); + return error("address index not enabled"); - if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) - return error("%s: unable to get txids for address"); + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex, start, end)) + return error("unable to get txids for address"); return true; } diff --git a/src/main.h b/src/main.h index 3378b24df5916..181f88993f16e 100644 --- a/src/main.h +++ b/src/main.h @@ -359,7 +359,7 @@ struct CAddressIndexKey { int blockHeight; unsigned int txindex; uint256 txhash; - size_t outindex; + size_t index; bool spending; size_t GetSerializeSize(int nType, int nVersion) const { @@ -373,7 +373,7 @@ struct CAddressIndexKey { ser_writedata32be(s, blockHeight); ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); - ser_writedata32(s, outindex); + ser_writedata32(s, index); char f = spending; ser_writedata8(s, f); } @@ -384,19 +384,19 @@ struct CAddressIndexKey { blockHeight = ser_readdata32be(s); txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); - outindex = ser_readdata32(s); + index = ser_readdata32(s); char f = ser_readdata8(s); spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex, bool isSpending) { + uint256 txid, size_t indexValue, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; - outindex = outputIndex; + index = indexValue; spending = isSpending; } @@ -410,7 +410,7 @@ struct CAddressIndexKey { blockHeight = 0; txindex = 0; txhash.SetNull(); - outindex = 0; + index = 0; spending = false; } @@ -419,14 +419,23 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; + bool includeHeight; + int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - return 21; + if (includeHeight) { + return 25; + } else { + return 21; + } } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); + if (includeHeight) { + ser_writedata32be(s, blockHeight); + } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -437,6 +446,14 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; + includeHeight = false; + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + includeHeight = true; } CAddressIndexIteratorKey() { @@ -446,6 +463,7 @@ struct CAddressIndexIteratorKey { void SetNull() { type = 0; hashBytes.SetNull(); + includeHeight = false; } }; @@ -580,7 +598,9 @@ class CScriptCheck }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 549e6c7a5b0f5..8685a327aba47 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,66 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("satoshis", it->second)); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("height", it->first.blockHeight)); + delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); + delta.push_back(Pair("type", (int)it->first.type)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressbalance(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 51235c184f4df..c955484626580 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 0be81809340be..fc57aa16b3f37 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 3aa6860583971..eb03b30e492b0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,16 +172,25 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector > &addressIndex) { +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start, int end) { boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + if (start > 0 && end > 0) { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + } else { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + } while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + if (end > 0 && key.second.blockHeight > end) { + break; + } CAmount nValue; if (pcursor->GetValue(nValue)) { addressIndex.push_back(make_pair(key.second, nValue)); diff --git a/src/txdb.h b/src/txdb.h index 7fefcaafafefa..4586d51f898d4 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,7 +62,9 @@ class CBlockTreeDB : public CDBWrapper bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); - bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); From 38a7d6d3234723d88be2b4a317e304ec9a8ada5e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 25 Mar 2016 14:52:45 -0400 Subject: [PATCH 19/53] rpc: optimize address txid queries --- src/rpcmisc.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8685a327aba47..8d939ffeaf3f4 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -567,24 +567,26 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::set txids; - std::vector > vtxids; + std::set > txids; + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); - if (txids.insert(txid).second) { - vtxids.push_back(std::make_pair(height, txid)); + + if (addresses.size() > 1) { + txids.insert(std::make_pair(height, txid)); + } else { + if (txids.insert(std::make_pair(height, txid)).second) { + result.push_back(txid); + } } } if (addresses.size() > 1) { - std::sort(vtxids.begin(), vtxids.end()); - } - - UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { - result.push_back(it->second); + for (std::set >::const_iterator it=txids.begin(); it!=txids.end(); it++) { + result.push_back(it->second); + } } return result; From 186e11fde7eac7765686b7e52655da7666866fb7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 28 Mar 2016 16:47:20 -0400 Subject: [PATCH 20/53] main: update address index during reorgs --- qa/rpc-tests/addressindex.py | 19 +++++++++++++-- src/main.cpp | 46 ++++++++++++++++++++++++++++++++++++ src/txdb.cpp | 7 ++++++ src/txdb.h | 1 + 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 4fc4cc557e361..a60c845b34ea7 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -118,10 +118,12 @@ def run_test(self): assert_equal(txidsmany[3], sent_txid) # Check that balances are correct + print "Testing balances..." balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") assert_equal(balance0["balance"], 45 * 100000000 + 21) # Check that balances are correct after spending + print "Testing balances after spending..." privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") @@ -158,14 +160,27 @@ def run_test(self): # Check that deltas are returned correctly deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) - balance3 = 0; + balance3 = 0 for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) - assert_equal(len(deltas), 1); + assert_equal(len(deltas), 1) + + # Check that indexes will be updated with a reorg + print "Testing reorg..." + + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + self.nodes[1].invalidateblock(best_hash) + self.nodes[2].invalidateblock(best_hash) + self.nodes[3].invalidateblock(best_hash) + self.sync_all() + + balance4 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance4, balance1) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 4b7c95afd4ec5..d4555b0ce5ad5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2079,6 +2079,52 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } + // undo address indexes + if (fAddressIndex) { + std::vector > addressIndex; + + for (unsigned int i = 0; i < block.vtx.size(); i++) { + const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); + + if (!tx.IsCoinBase()) { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else { + continue; + } + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + + if (!pblocktree->EraseAddressIndex(addressIndex)) { + return AbortNode(state, "Failed to delete address index"); + } + } + + // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); diff --git a/src/txdb.cpp b/src/txdb.cpp index eb03b30e492b0..2128c181f27d1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,6 +172,13 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Erase(make_pair(DB_ADDRESSINDEX, it->first)); + return WriteBatch(batch); +} + bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { diff --git a/src/txdb.h b/src/txdb.h index 4586d51f898d4..93b5c0f5c14db 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,6 +62,7 @@ class CBlockTreeDB : public CDBWrapper bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); + bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); From 8597289d8b1cef8ebc2895f383f0ce9ac9666b00 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 09:13:31 -0400 Subject: [PATCH 21/53] main: fix order of address index when disconnecting block --- src/main.cpp | 59 ++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d4555b0ce5ad5..6616c290ed181 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2041,11 +2041,32 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) return error("DisconnectBlock(): block and undo data inconsistent"); + std::vector > addressIndex; + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = block.vtx[i]; uint256 hash = tx.GetHash(); + if (fAddressIndex) { + + for (unsigned int k = tx.vout.size(); k-- > 0;) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + // Check that all outputs are available and match the outputs in the block itself // exactly. { @@ -2075,56 +2096,30 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxInUndo &undo = txundo.vprevout[j]; if (!ApplyTxInUndo(undo, view, out)) fClean = false; - } - } - } - - // undo address indexes - if (fAddressIndex) { - std::vector > addressIndex; - - for (unsigned int i = 0; i < block.vtx.size(); i++) { - const CTransaction &tx = block.vtx[i]; - const uint256 txhash = tx.GetHash(); - if (!tx.IsCoinBase()) { - for (size_t j = 0; j < tx.vin.size(); j++) { + if (fAddressIndex) { const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else { continue; } } } - - for (unsigned int k = 0; k < tx.vout.size(); k++) { - const CTxOut &out = tx.vout[k]; - - if (out.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else { - continue; - } - - } - } + } + + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } } - // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); From 0b42ba227af7fe537e7bad095c0015f9201d9cca Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 15:17:30 -0400 Subject: [PATCH 22/53] main: index unspent outputs by address --- qa/rpc-tests/addressindex.py | 6 +++ src/main.cpp | 46 +++++++++++++++++- src/main.h | 91 +++++++++++++++++++++++++++++++++++- src/rpcmisc.cpp | 50 ++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 39 ++++++++++++++++ src/txdb.h | 5 ++ 8 files changed, 236 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a60c845b34ea7..5448af6c6346a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -169,6 +169,12 @@ def run_test(self): deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) + # Check that unspent outputs can be queried + print "Testing utxos..." + utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos), 2) + assert_equal(utxos[0]["satoshis"], 5000000000) + # Check that indexes will be updated with a reorg print "Testing reorg..." diff --git a/src/main.cpp b/src/main.cpp index 6616c290ed181..dbbeef8f9b2f7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1463,6 +1463,18 @@ bool GetAddressIndex(uint160 addressHash, int type, return true; } +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs) +{ + if (!fAddressIndex) + return error("address index not enabled"); + + if (!pblocktree->ReadAddressUnspentIndex(addressHash, type, unspentOutputs)) + return error("unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2389,6 +2401,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; + std::vector > addressUnspentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2426,10 +2439,21 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + } else { continue; } @@ -2463,10 +2487,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else { continue; } @@ -2524,9 +2560,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); - if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) + if (fAddressIndex) { + if (!pblocktree->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); + } + + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } + } if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) diff --git a/src/main.h b/src/main.h index 181f88993f16e..0964e6dc36058 100644 --- a/src/main.h +++ b/src/main.h @@ -353,6 +353,93 @@ struct CTimestampIndexKey { } }; +struct CAddressUnspentKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; + uint256 txhash; + size_t index; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + } + + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t indexValue) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; + txhash = txid; + index = indexValue; + } + + CAddressUnspentKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; + txhash.SetNull(); + index = 0; + } +}; + +struct CAddressUnspentValue { + CAmount satoshis; + CScript script; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(satoshis); + READWRITE(*(CScriptBase*)(&script)); + } + + CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + satoshis = sats; + script = scriptPubKey; + } + + CAddressUnspentValue() { + SetNull(); + } + + void SetNull() { + satoshis = 0; + script = CScript(); + } + + bool IsNull() const { + return (satoshis == 0); + } + +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -363,7 +450,7 @@ struct CAddressIndexKey { bool spending; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 66; } template void Serialize(Stream& s, int nType, int nVersion) const { @@ -601,6 +688,8 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8d939ffeaf3f4..1eeded608ff46 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,56 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > unspentOutputs; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressUnspent((*it).first, (*it).second, unspentOutputs)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { + UniValue output(UniValue::VOBJ); + output.push_back(Pair("addressType", (int)it->first.type)); + output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + output.push_back(Pair("txid", it->first.txhash.GetHex())); + output.push_back(Pair("height", it->first.blockHeight)); + output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); + output.push_back(Pair("satoshis", it->second.satoshis)); + result.push_back(output); + } + + return result; +} + UniValue getaddressdeltas(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1 || !params[0].isObject()) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index c955484626580..39af0513b5da6 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index fc57aa16b3f37..5ad147dec7f08 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 2128c181f27d1..8e485e9eb6620 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; @@ -165,6 +166,44 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_ADDRESSUNSPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_ADDRESSUNSPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &unspentOutputs) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSUNSPENTINDEX, CAddressIndexIteratorKey(type, addressHash))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSUNSPENTINDEX && key.second.hashBytes == addressHash) { + CAddressUnspentValue nValue; + if (pcursor->GetValue(nValue)) { + unspentOutputs.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address unspent value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) diff --git a/src/txdb.h b/src/txdb.h index 93b5c0f5c14db..547a48fa8412e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,8 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressUnspentKey; +struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; @@ -61,6 +63,9 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool UpdateAddressUnspentIndex(const std::vector >&vect); + bool ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &vect); bool WriteAddressIndex(const std::vector > &vect); bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, From 08836972c093eb137e1c11eb9596e7d12d600332 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 17:46:59 -0400 Subject: [PATCH 23/53] rpc: fix argument check for getaddressutxos --- src/rpcmisc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1eeded608ff46..2c3b7e6d00bb4 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -437,7 +437,7 @@ bool getAddressesFromParams(const UniValue& params, std::vector Date: Tue, 29 Mar 2016 17:56:57 -0400 Subject: [PATCH 24/53] main: update unspent address index during reorgs --- qa/rpc-tests/addressindex.py | 6 ++++++ src/main.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 5448af6c6346a..d051d170a944a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -174,6 +174,7 @@ def run_test(self): utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) assert_equal(len(utxos), 2) assert_equal(utxos[0]["satoshis"], 5000000000) + assert_equal(utxos[1]["satoshis"], 4899977160) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -188,6 +189,11 @@ def run_test(self): balance4 = self.nodes[1].getaddressbalance(address2) assert_equal(balance4, balance1) + utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos2), 2) + assert_equal(utxos2[0]["satoshis"], 5000000000) + assert_equal(utxos2[1]["satoshis"], 5000000000) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index dbbeef8f9b2f7..4d6872d2d41ca 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2054,6 +2054,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI return error("DisconnectBlock(): block and undo data inconsistent"); std::vector > addressIndex; + std::vector > addressUnspentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -2067,10 +2068,22 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else { continue; } @@ -2113,10 +2126,23 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + } else { continue; } @@ -2130,6 +2156,9 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } } // move best block pointer to prevout block From 21c675855fa241c13771e03d9098939355c10b0e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 12:50:10 -0400 Subject: [PATCH 25/53] main: don't undo indexes when verifying blocks at startup --- src/main.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 4d6872d2d41ca..b061383efa07d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2152,6 +2152,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } + // move best block pointer to prevout block + view.SetBestBlock(pindex->pprev->GetBlockHash()); + + if (pfClean) { + *pfClean = fClean; + return true; + } + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); @@ -2161,14 +2169,6 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } - // move best block pointer to prevout block - view.SetBestBlock(pindex->pprev->GetBlockHash()); - - if (pfClean) { - *pfClean = fClean; - return true; - } - return fClean; } From d0483c9aa09a45f9e87902b9e023f977a3006830 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 15:12:19 -0400 Subject: [PATCH 26/53] main: remove spent address utxo indexes --- qa/rpc-tests/addressindex.py | 8 +++----- src/main.cpp | 18 ++++++++++-------- src/main.h | 23 +++++------------------ src/rpcmisc.cpp | 1 - 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d051d170a944a..703f3029ada91 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -172,9 +172,8 @@ def run_test(self): # Check that unspent outputs can be queried print "Testing utxos..." utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos), 2) - assert_equal(utxos[0]["satoshis"], 5000000000) - assert_equal(utxos[1]["satoshis"], 4899977160) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]["satoshis"], change_amount) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -190,9 +189,8 @@ def run_test(self): assert_equal(balance4, balance1) utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos2), 2) + assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) - assert_equal(utxos2[1]["satoshis"], 5000000000) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index b061383efa07d..8a8189c97581c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2073,7 +2073,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2082,7 +2082,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else { continue; @@ -2123,6 +2123,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI fClean = false; if (fAddressIndex) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2131,7 +2132,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -2141,7 +2142,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else { continue; @@ -2465,6 +2466,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fAddressIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2473,7 +2475,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); @@ -2481,7 +2483,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else { continue; @@ -2521,7 +2523,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2530,7 +2532,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else { continue; diff --git a/src/main.h b/src/main.h index 0964e6dc36058..15ade067a6bc0 100644 --- a/src/main.h +++ b/src/main.h @@ -356,21 +356,16 @@ struct CTimestampIndexKey { struct CAddressUnspentKey { unsigned int type; uint160 hashBytes; - int blockHeight; - unsigned int txindex; uint256 txhash; size_t index; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 57; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - // Heights are stored big-endian for key sorting in LevelDB - ser_writedata32be(s, blockHeight); - ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, index); } @@ -378,18 +373,13 @@ struct CAddressUnspentKey { void Unserialize(Stream& s, int nType, int nVersion) { type = ser_readdata8(s); hashBytes.Unserialize(s, nType, nVersion); - blockHeight = ser_readdata32be(s); - txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); index = ser_readdata32(s); } - CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t indexValue) { + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, uint256 txid, size_t indexValue) { type = addressType; hashBytes = addressHash; - blockHeight = height; - txindex = blockindex; txhash = txid; index = indexValue; } @@ -401,8 +391,6 @@ struct CAddressUnspentKey { void SetNull() { type = 0; hashBytes.SetNull(); - blockHeight = 0; - txindex = 0; txhash.SetNull(); index = 0; } @@ -430,14 +418,13 @@ struct CAddressUnspentValue { } void SetNull() { - satoshis = 0; - script = CScript(); + satoshis = -1; + script.clear(); } bool IsNull() const { - return (satoshis == 0); + return (satoshis == -1); } - }; struct CAddressIndexKey { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 2c3b7e6d00bb4..832d55f1af552 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -475,7 +475,6 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("addressType", (int)it->first.type)); output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("height", it->first.blockHeight)); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); From 1bd65a5c4bfebcc9a87a0f69b10b3711f8c79ec7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 16:42:37 -0400 Subject: [PATCH 27/53] main: sort address index utxos by height --- qa/rpc-tests/addressindex.py | 15 +++++++++++++++ src/main.cpp | 8 ++++---- src/main.h | 6 +++++- src/rpcmisc.cpp | 8 +++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 703f3029ada91..ad79c741635cc 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -192,6 +192,21 @@ def run_test(self): assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) + # Check sorting of utxos + self.nodes[2].generate(150) + + txidsort1 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + txidsort2 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + self.sync_all() + + utxos3 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos3), 3) + assert_equal(utxos3[0]["height"], 114) + assert_equal(utxos3[1]["height"], 264) + assert_equal(utxos3[2]["height"], 265) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 8a8189c97581c..b4c7d8e248000 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2132,7 +2132,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -2142,7 +2142,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else { continue; @@ -2523,7 +2523,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2532,7 +2532,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else { continue; diff --git a/src/main.h b/src/main.h index 15ade067a6bc0..90ed55e9294cd 100644 --- a/src/main.h +++ b/src/main.h @@ -399,6 +399,7 @@ struct CAddressUnspentKey { struct CAddressUnspentValue { CAmount satoshis; CScript script; + int blockHeight; ADD_SERIALIZE_METHODS; @@ -406,11 +407,13 @@ struct CAddressUnspentValue { inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { READWRITE(satoshis); READWRITE(*(CScriptBase*)(&script)); + READWRITE(blockHeight); } - CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { satoshis = sats; script = scriptPubKey; + blockHeight = height; } CAddressUnspentValue() { @@ -420,6 +423,7 @@ struct CAddressUnspentValue { void SetNull() { satoshis = -1; script.clear(); + blockHeight = 0; } bool IsNull() const { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 832d55f1af552..ec0774660c492 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -431,8 +431,11 @@ bool getAddressesFromParams(const UniValue& params, std::vector a, + std::pair b) { + return a.second.blockHeight < b.second.blockHeight; } UniValue getaddressutxos(const UniValue& params, bool fHelp) @@ -468,6 +471,8 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) } } + std::sort(unspentOutputs.begin(), unspentOutputs.end(), heightSort); + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { @@ -478,6 +483,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); + output.push_back(Pair("height", it->second.blockHeight)); result.push_back(output); } From b66eff47cff0c4b1135276b3cd30f108ea7fee54 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 4 Apr 2016 16:37:43 -0400 Subject: [PATCH 28/53] main: mempool address index --- qa/rpc-tests/addressindex.py | 40 ++++++++++++++++++- src/addressindex.h | 77 ++++++++++++++++++++++++++++++++++++ src/main.cpp | 3 ++ src/rpcmisc.cpp | 46 +++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txmempool.cpp | 75 +++++++++++++++++++++++++++++++++++ src/txmempool.h | 12 ++++++ 8 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 src/addressindex.h diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index ad79c741635cc..d28cd393fb4b3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -7,6 +7,7 @@ # Test addressindex generation and fetching # +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * @@ -25,7 +26,7 @@ def setup_network(self): self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-addressindex"])) self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -207,6 +208,43 @@ def run_test(self): assert_equal(utxos3[1]["height"], 264) assert_equal(utxos3[2]["height"], 265) + # Check mempool indexing + print "Testing mempool indexing..." + + privKey3 = "cVfUn53hAbRrDEuMexyfgDpZPhF7KqXpS8UZevsyTDaugB7HZ3CD" + address3 = "mw4ynwhS7MmrQ27hr82kgqu7zryNDK26JB" + addressHash3 = "aa9872b5bbcdb511d89e0e11aa27da73fd2c3f50".decode("hex") + scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[2].listunspent() + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey3)] + tx.rehash() + signed_tx = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid1 = self.nodes[2].sendrawtransaction(signed_tx["hex"], True) + time.sleep(2) + + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] + amount = unspent[1]["amount"] * 100000000 + tx2.vout = [CTxOut(amount, scriptPubKey3)] + tx2.rehash() + signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) + time.sleep(2) + + mempool = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool), 2) + assert_equal(mempool[0]["txid"], memtxid1) + assert_equal(mempool[1]["txid"], memtxid2) + + self.nodes[2].generate(1); + self.sync_all(); + mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool2), 0) + print "Passed\n" diff --git a/src/addressindex.h b/src/addressindex.h new file mode 100644 index 0000000000000..b6e3587c1f6cf --- /dev/null +++ b/src/addressindex.h @@ -0,0 +1,77 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRESSINDEX_H +#define BITCOIN_ADDRESSINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CMempoolAddressDelta +{ + int64_t time; + CAmount amount; + uint256 prevhash; + unsigned int prevout; + + CMempoolAddressDelta(int64_t t, CAmount a, uint256 hash, unsigned int out) { + time = t; + amount = a; + prevhash = hash; + prevout = out; + } + + CMempoolAddressDelta(int64_t t, CAmount a) { + time = t; + amount = a; + prevhash.SetNull(); + prevout = 0; + } +}; + +struct CMempoolAddressDeltaKey +{ + int type; + uint160 addressBytes; + uint256 txhash; + unsigned int index; + bool spending; + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, bool s) { + type = addressType; + addressBytes = addressHash; + txhash = hash; + index = i; + spending = s; + } + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash) { + type = addressType; + addressBytes = addressHash; + txhash.SetNull(); + index = 0; + } +}; + +struct CMempoolAddressDeltaKeyCompare +{ + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + if (a.type == b.type) { + if (a.addressBytes == b.addressBytes) { + if (a.txhash == b.txhash) { + return a.index < b.index; + } else { + return a.txhash < b.txhash; + } + } else { + return a.addressBytes < b.addressBytes; + } + } else { + return a.type < b.type; + } + } +}; + +#endif // BITCOIN_ADDRESSINDEX_H diff --git a/src/main.cpp b/src/main.cpp index b4c7d8e248000..5af73848bf44a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1414,6 +1414,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState &state, const C // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + if (fAddressIndex) { + pool.addAddressIndex(entry, view); + } // trim mempool and check if tx was trimmed if (!fOverrideMempoolLimit) { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index ec0774660c492..f06baa04278fb 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -11,6 +11,7 @@ #include "netbase.h" #include "rpcserver.h" #include "timedata.h" +#include "txmempool.h" #include "util.h" #include "utilstrencodings.h" #ifdef ENABLE_WALLET @@ -438,6 +439,51 @@ bool heightSort(std::pair a, return a.second.blockHeight < b.second.blockHeight; } +bool timestampSort(std::pair a, + std::pair b) { + return a.second.time < b.second.time; +} + +UniValue getaddressmempool(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressmempool\n" + "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > indexes; + + if (!mempool.getAddressIndex(addresses, indexes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + + std::sort(indexes.begin(), indexes.end(), timestampSort); + + UniValue result(UniValue::VARR); + + for (std::vector >::iterator it = indexes.begin(); + it != indexes.end(); it++) { + + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("addressType", (int)it->first.type)); + delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("satoshis", it->second.amount)); + delta.push_back(Pair("timestamp", it->second.time)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressutxos(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 39af0513b5da6..1d5399df39d25 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressmempool", &getaddressmempool, false }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 5ad147dec7f08..74ecd11918c87 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressmempool(const UniValue& params, bool fHelp); extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 5f814749b71ff..adbe662aba022 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -422,6 +422,80 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, return true; } +void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + std::pair ret; + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } + } + + mapAddressInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getAddressIndex(std::vector > &addresses, + std::vector > &results) +{ + LOCK(cs); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + addressDeltaMap::iterator ait = mapAddress.lower_bound(CMempoolAddressDeltaKey((*it).second, (*it).first)); + while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first) { + results.push_back(*ait); + ait++; + } + } + return true; +} + +bool CTxMemPool::removeAddressIndex(const uint256 txhash) +{ + LOCK(cs); + addressDeltaMapInserted::iterator it = mapAddressInserted.find(txhash); + + if (it != mapAddressInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapAddress.erase(*mit); + } + mapAddressInserted.erase(it); + } + + return true; +} + void CTxMemPool::removeUnchecked(txiter it) { const uint256 hash = it->GetTx().GetHash(); @@ -435,6 +509,7 @@ void CTxMemPool::removeUnchecked(txiter it) mapTx.erase(it); nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); + removeAddressIndex(hash); } // Calculates descendants of entry that are not already in setDescendants, and adds to diff --git a/src/txmempool.h b/src/txmempool.h index 5997346b022bb..57091a02e62da 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -9,6 +9,7 @@ #include #include +#include "addressindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -419,6 +420,12 @@ class CTxMemPool typedef std::map txlinksMap; txlinksMap mapLinks; + typedef std::map addressDeltaMap; + addressDeltaMap mapAddress; + + typedef std::map > addressDeltaMapInserted; + addressDeltaMapInserted mapAddressInserted; + void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); @@ -450,6 +457,11 @@ class CTxMemPool bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); + void addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getAddressIndex(std::vector > &addresses, + std::vector > &results); + bool removeAddressIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed); From d99f17de1a708f67e473298f638cba04596f6c98 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 10:49:11 -0400 Subject: [PATCH 29/53] rpc: give back base58 encoded address format in utxos --- src/rpcmisc.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index f06baa04278fb..c79a4ccddd402 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -523,8 +523,16 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); - output.push_back(Pair("addressType", (int)it->first.type)); - output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + std::string address; + if (it->first.type == 2) { + address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); + } else if (it->first.type == 1) { + address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + + output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); From 7c68235cf4a9c51b079660338e1fd9490dfa0859 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:41:19 -0400 Subject: [PATCH 30/53] main: include timestampindex in help --- src/init.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/init.cpp b/src/init.cpp index 16598ada44e39..7f10a32e4fe0c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -351,6 +351,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); From 4678f2d4389f9cce467f51a55f1da7d0e8debd63 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 12:35:25 -0400 Subject: [PATCH 31/53] build: add addressindex.h to make --- src/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile.am b/src/Makefile.am index 52316a9fd7b7f..9632d65676ca9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -79,6 +79,7 @@ endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ + addressindex.h \ addrman.h \ alert.h \ amount.h \ From 8636f364770089418cb613403cbe1288da9842b6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 13:52:40 -0400 Subject: [PATCH 32/53] rpc: cast indexes to ints --- src/rpcmisc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index c79a4ccddd402..4b0d04ae94056 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -534,7 +534,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("outputIndex", (int)it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); output.push_back(Pair("height", it->second.blockHeight)); @@ -594,7 +594,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); - delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); delta.push_back(Pair("type", (int)it->first.type)); From e3d9207e5acd3ec53176d8f0d455a541419429a5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:53:38 -0400 Subject: [PATCH 33/53] main: add spentindex option --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/spentindex.py | 73 ++++++++++++++++++++++++++++++++++ src/init.cpp | 1 + src/main.cpp | 78 +++++++++++++++++++++++++++++-------- src/main.h | 65 +++++++++++++++++++++++++++++++ src/rpcmisc.cpp | 39 +++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 17 ++++++++ src/txdb.h | 4 ++ 10 files changed, 263 insertions(+), 17 deletions(-) create mode 100755 qa/rpc-tests/spentindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 73bfe1e28b23b..9c07091f7f8d9 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -99,6 +99,7 @@ 'reindex.py', 'addressindex.py', 'timestampindex.py', + 'spentindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py new file mode 100755 index 0000000000000..0fe98dcbd5e3b --- /dev/null +++ b/qa/rpc-tests/spentindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class SpentIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + # Check that + print "Testing spent index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) + assert_equal(info["txid"], txid) + assert_equal(info["index"], 0) + + print "Passed\n" + + +if __name__ == '__main__': + SpentIndexTest().main() diff --git a/src/init.cpp b/src/init.cpp index 7f10a32e4fe0c..3893500926d6d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -352,6 +352,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); + strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); diff --git a/src/main.cpp b/src/main.cpp index 5af73848bf44a..e96d58aa6eb38 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -68,6 +68,7 @@ bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; bool fTimestampIndex = false; +bool fSpentIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1454,6 +1455,17 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + if (!fSpentIndex) + return error("spent index not enabled"); + + if (!pblocktree->ReadSpentIndex(key, value)) + return error("unable to get spent info"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { @@ -2058,6 +2070,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -2125,8 +2138,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!ApplyTxInUndo(undo, view, out)) fClean = false; + const CTxIn input = tx.vin[j]; + + if (fSpentIndex) { + // undo and delete the spent index + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue())); + } + if (fAddressIndex) { - const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2151,6 +2170,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI continue; } } + } } } @@ -2435,6 +2455,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2466,31 +2487,43 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-txns-nonfinal"); } - if (fAddressIndex) + if (fAddressIndex || fSpentIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + if (fSpentIndex) { + // add the spent index to determine the txid and input that spent an output + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); + } - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + if (fAddressIndex) { - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - } else { - continue; + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + + } else { + continue; + } } + } } @@ -2604,6 +2637,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } + if (fSpentIndex) + if (!pblocktree->UpdateSpentIndex(spentIndex)) + return AbortNode(state, "Failed to write transaction index"); + if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); @@ -4007,6 +4044,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("timestampindex", fTimestampIndex); LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Check whether we have a spent index + pblocktree->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -4176,6 +4217,9 @@ bool InitBlockIndex(const CChainParams& chainparams) fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); pblocktree->WriteFlag("timestampindex", fTimestampIndex); + fSpentIndex = GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); + pblocktree->WriteFlag("spentindex", fSpentIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 90ed55e9294cd..7e54909c16e38 100644 --- a/src/main.h +++ b/src/main.h @@ -113,6 +113,7 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; +static const bool DEFAULT_SPENTINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -292,6 +293,69 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h) { + txid = t; + inputIndex = i; + blockHeight = h; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + struct CTimestampIndexIteratorKey { unsigned int timestamp; @@ -676,6 +740,7 @@ class CScriptCheck }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 4b0d04ae94056..b31864993bdcf 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -701,3 +701,42 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) return result; } + +UniValue getspentinfo(const UniValue& params, bool fHelp) +{ + + if (fHelp || params.size() != 1 || !params[0].isObject()) + throw runtime_error( + "getspentinfo\n" + "\nReturns the txid and index where an output is spent.\n" + "\nResult\n" + "{\n" + " \"txid\" (string) The transaction id\n" + " \"index\" (number) The spending input index\n" + " ,...\n" + "}\n" + ); + + UniValue txidValue = find_value(params[0].get_obj(), "txid"); + UniValue indexValue = find_value(params[0].get_obj(), "index"); + + if (!txidValue.isStr() || !indexValue.isNum()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid txid or index"); + } + + uint256 txid = ParseHashV(txidValue, "txid"); + int outputIndex = indexValue.get_int(); + + CSpentIndexKey key(txid, outputIndex); + CSpentIndexValue value; + + if (!GetSpentIndex(key, value)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); + } + + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("txid", value.txid.GetHex())); + obj.push_back(Pair("index", (int)value.inputIndex)); + + return obj; +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 1d5399df39d25..6717592fa9e4b 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,6 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, + { "blockchain", "getspentinfo", &getspentinfo, true }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 74ecd11918c87..581370c26bf4d 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -271,6 +271,7 @@ extern UniValue verifychain(const UniValue& params, bool fHelp); extern UniValue getchaintips(const UniValue& params, bool fHelp); extern UniValue invalidateblock(const UniValue& params, bool fHelp); extern UniValue reconsiderblock(const UniValue& params, bool fHelp); +extern UniValue getspentinfo(const UniValue& params, bool fHelp); bool StartRPC(); void InterruptRPC(); diff --git a/src/txdb.cpp b/src/txdb.cpp index 8e485e9eb6620..d3a29d5fd2be9 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -24,6 +24,7 @@ static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; +static const char DB_SPENTINDEX = 'p'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -166,6 +167,22 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { + return Read(make_pair(DB_SPENTINDEX, key), value); +} + +bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_SPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_SPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index 547a48fa8412e..62b115707b236 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,6 +23,8 @@ struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; +struct CSpentIndexKey; +struct CSpentIndexValue; class uint256; //! -dbcache default (MiB) @@ -63,6 +65,8 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool UpdateSpentIndex(const std::vector >&vect); bool UpdateAddressUnspentIndex(const std::vector >&vect); bool ReadAddressUnspentIndex(uint160 addressHash, int type, std::vector > &vect); From b752fbe09e73e22101c151f139c2001f08d9bd00 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 12:31:21 -0400 Subject: [PATCH 34/53] rpc: include spent info if spentindex enabled with getrawtransaction verbose --- qa/rpc-tests/spentindex.py | 8 +++++++- src/main.cpp | 2 +- src/rpcrawtransaction.cpp | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 0fe98dcbd5e3b..44ba286609d2b 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -27,7 +27,7 @@ def setup_network(self): self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) # Nodes 2/3 are used for testing self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) - self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex", "-txindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) @@ -62,10 +62,16 @@ def run_test(self): self.nodes[0].generate(1) self.sync_all() + # Check that the spentinfo works standalone info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + # Check that verbose raw transaction includes spent info + txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index e96d58aa6eb38..d750477e92b10 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1458,7 +1458,7 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { if (!fSpentIndex) - return error("spent index not enabled"); + return false; if (!pblocktree->ReadSpentIndex(key, value)) return error("unable to get spent info"); diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 6ab1807d46373..0e8df7faedadc 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -61,7 +61,8 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); entry.push_back(Pair("version", tx.nVersion)); entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); @@ -91,6 +92,15 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(txout.scriptPubKey, o, true); out.push_back(Pair("scriptPubKey", o)); + + // Add spent information if spentindex is enabled + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(txid, i); + if (GetSpentIndex(spentKey, spentInfo)) { + out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); + out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + } + vout.push_back(out); } entry.push_back(Pair("vout", vout)); From abe40712ce25f2383e60db9bfeb59d02473502ec Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:31:19 -0400 Subject: [PATCH 35/53] rpc: include height in spentinfo --- qa/rpc-tests/spentindex.py | 2 ++ src/rpcmisc.cpp | 1 + src/rpcrawtransaction.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 44ba286609d2b..6669df462f8de 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -66,11 +66,13 @@ def run_test(self): info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + assert_equal(info["height"], 106) # Check that verbose raw transaction includes spent info txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index b31864993bdcf..fc55f3df2f969 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -737,6 +737,7 @@ UniValue getspentinfo(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.push_back(Pair("txid", value.txid.GetHex())); obj.push_back(Pair("index", (int)value.inputIndex)); + obj.push_back(Pair("height", value.blockHeight)); return obj; } diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 0e8df7faedadc..6fbec480edaa5 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -99,6 +99,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (GetSpentIndex(spentKey, spentInfo)) { out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + out.push_back(Pair("spentHeight", spentInfo.blockHeight)); } vout.push_back(out); From 96d8307b559535fcf0809c4734b83895a5f8ab43 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 16:04:10 -0400 Subject: [PATCH 36/53] rpc: query txids for addresses within block height range --- qa/rpc-tests/addressindex.py | 10 ++++++++++ src/rpcmisc.cpp | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d28cd393fb4b3..3ef5bcf675be3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -84,6 +84,16 @@ def run_test(self): assert_equal(txidsb[1], txidb1) assert_equal(txidsb[2], txidb2) + # Check that limiting by height works + chain_height = self.nodes[1].getblockcount() + height_txids = self.nodes[1].getaddresstxids({ + "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], + "start": 111, + "end": 111 + }) + assert_equal(len(height_txids), 1) + assert_equal(height_txids[0], txidb2) + # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) assert_equal(len(multitxids), 6) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index fc55f3df2f969..9e5266e31b2aa 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -668,11 +668,28 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + int start = 0; + int end = 0; + if (params[0].isObject()) { + UniValue startValue = find_value(params[0].get_obj(), "start"); + UniValue endValue = find_value(params[0].get_obj(), "end"); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = startValue.get_int(); + } + } + std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 94ea69a4d59852b06c89b484a3dab324ff0c39fa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 20:33:18 -0400 Subject: [PATCH 37/53] rpc: fix issue with querying txids by block heights --- qa/rpc-tests/addressindex.py | 11 ++++---- src/main.h | 50 +++++++++++++++++++++++++----------- src/rpcmisc.cpp | 2 +- src/txdb.cpp | 2 +- src/txdb.h | 1 + 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 3ef5bcf675be3..e45efdbda378b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -85,14 +85,15 @@ def run_test(self): assert_equal(txidsb[2], txidb2) # Check that limiting by height works - chain_height = self.nodes[1].getblockcount() + print "Testing querying txids by range of block heights.." height_txids = self.nodes[1].getaddresstxids({ "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], - "start": 111, - "end": 111 + "start": 105, + "end": 110 }) - assert_equal(len(height_txids), 1) - assert_equal(height_txids[0], txidb2) + assert_equal(len(height_txids), 2) + assert_equal(height_txids[0], txidb0) + assert_equal(height_txids[1], txidb1) # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) diff --git a/src/main.h b/src/main.h index 7e54909c16e38..6c685e68ff267 100644 --- a/src/main.h +++ b/src/main.h @@ -561,23 +561,14 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - bool includeHeight; - int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - if (includeHeight) { - return 25; - } else { - return 21; - } + return 21; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - if (includeHeight) { - ser_writedata32be(s, blockHeight); - } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -588,24 +579,53 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; - includeHeight = false; } - CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { + CAddressIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + } +}; + +struct CAddressIndexIteratorHeightKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 25; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + ser_writedata32be(s, blockHeight); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + } + + CAddressIndexIteratorHeightKey(unsigned int addressType, uint160 addressHash, int height) { type = addressType; hashBytes = addressHash; blockHeight = height; - includeHeight = true; } - CAddressIndexIteratorKey() { + CAddressIndexIteratorHeightKey() { SetNull(); } void SetNull() { type = 0; hashBytes.SetNull(); - includeHeight = false; + blockHeight = 0; } }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9e5266e31b2aa..91a45813a05d6 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -675,7 +675,7 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) UniValue endValue = find_value(params[0].get_obj(), "end"); if (startValue.isNum() && endValue.isNum()) { start = startValue.get_int(); - end = startValue.get_int(); + end = endValue.get_int(); } } diff --git a/src/txdb.cpp b/src/txdb.cpp index d3a29d5fd2be9..8a4d71578eae8 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -242,7 +242,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, boost::scoped_ptr pcursor(NewIterator()); if (start > 0 && end > 0) { - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorHeightKey(type, addressHash, start))); } else { pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); } diff --git a/src/txdb.h b/src/txdb.h index 62b115707b236..14d501278f23e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -21,6 +21,7 @@ struct CAddressUnspentKey; struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CAddressIndexIteratorHeightKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; struct CSpentIndexKey; From 04da9306c62c062536a309e99178c9b742a01560 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 23:25:55 -0400 Subject: [PATCH 38/53] build: fix darwin build --- src/addressindex.h | 2 +- src/txdb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/addressindex.h b/src/addressindex.h index b6e3587c1f6cf..43b49dca9b1ce 100644 --- a/src/addressindex.h +++ b/src/addressindex.h @@ -57,7 +57,7 @@ struct CMempoolAddressDeltaKey struct CMempoolAddressDeltaKeyCompare { - bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) const { if (a.type == b.type) { if (a.addressBytes == b.addressBytes) { if (a.txhash == b.txhash) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 8a4d71578eae8..48150198f3a34 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -271,7 +271,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey ×tampIndex) { CDBBatch batch(&GetObfuscateKey()); - batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), NULL); + batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), 0); return WriteBatch(batch); } From 5c3cf5f631a427fbe05dd9edc10bda6ddf02358a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:07:01 -0400 Subject: [PATCH 39/53] rpc: include prevhash and prevout information for spending deltas --- src/rpcmisc.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 91a45813a05d6..05455fb8bebd9 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -478,6 +478,10 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); delta.push_back(Pair("timestamp", it->second.time)); + if (it->second.amount < 0) { + delta.push_back(Pair("prevtxid", it->second.prevhash.GetHex())); + delta.push_back(Pair("prevout", (int)it->second.prevout)); + } result.push_back(delta); } From 28f9ae7853b0f335334b029029fe671eb59a886f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:23:52 -0400 Subject: [PATCH 40/53] test: test for getaddressmempool prevhash and prevout values --- qa/rpc-tests/addressindex.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index e45efdbda378b..c6910cd1ee392 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -256,6 +256,20 @@ def run_test(self): mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) assert_equal(len(mempool2), 0) + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0))] + tx.vout = [CTxOut(amount - 10000, scriptPubKey2)] + tx.rehash() + self.nodes[2].importprivkey(privKey3) + signed_tx3 = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid3 = self.nodes[2].sendrawtransaction(signed_tx3["hex"], True) + time.sleep(2) + + mempool3 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool3), 1) + assert_equal(mempool3[0]["prevtxid"], memtxid2) + assert_equal(mempool3[0]["prevout"], 0) + print "Passed\n" From 8391ff0b0ab51f25d0930084ce1a5ccef7b6eba0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:59:51 -0400 Subject: [PATCH 41/53] rpc: include base58check encoded address in results --- qa/rpc-tests/addressindex.py | 2 ++ src/rpcmisc.cpp | 37 +++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c6910cd1ee392..7ec00cc4f5ba8 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -176,6 +176,7 @@ def run_test(self): for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) + assert_equal(deltas[0]["address"], address2) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) @@ -250,6 +251,7 @@ def run_test(self): assert_equal(len(mempool), 2) assert_equal(mempool[0]["txid"], memtxid1) assert_equal(mempool[1]["txid"], memtxid2) + assert_equal(mempool[0]["address"], address3) self.nodes[2].generate(1); self.sync_all(); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 05455fb8bebd9..8f5a741e28a19 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -398,6 +398,18 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } +bool getAddressFromIndex(const int &type, const uint160 &hash, std::string &address) +{ + if (type == 2) { + address = CBitcoinAddress(CScriptID(hash)).ToString(); + } else if (type == 1) { + address = CBitcoinAddress(CKeyID(hash)).ToString(); + } else { + return false; + } + return true; +} + bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { if (params[0].isStr()) { @@ -471,9 +483,13 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) for (std::vector >::iterator it = indexes.begin(); it != indexes.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.addressBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); - delta.push_back(Pair("addressType", (int)it->first.type)); - delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("address", address)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); @@ -528,11 +544,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); std::string address; - if (it->first.type == 2) { - address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); - } else if (it->first.type == 1) { - address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); - } else { + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); } @@ -561,8 +573,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " \"txid\" (string) The related txid\n" " \"index\" (number) The related input or output index\n" " \"height\" (number) The block height\n" - " \"hash\" (string) The address hash\n" - " \"type\" (number) The address type 0 for pubkeyhash 1 for scripthash\n" + " \"address\" (string) The base58check encoded address\n" " }\n" "]\n" ); @@ -595,13 +606,17 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); - delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); - delta.push_back(Pair("type", (int)it->first.type)); + delta.push_back(Pair("address", address)); result.push_back(delta); } From 98f8fdddc858f86e48d09b4b1a289749dc66bda2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 16:07:42 -0400 Subject: [PATCH 42/53] rpc: optional "start" and "end" params for getaddressdeltas --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 24 +++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 7ec00cc4f5ba8..a07d4de1e0a6b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -178,6 +178,10 @@ def run_test(self): assert_equal(balance3, change_amount) assert_equal(deltas[0]["address"], address2) + # Check that entire range will be queried + deltasAll = self.nodes[1].getaddressdeltas({"addresses": [address2]}) + assert_equal(len(deltasAll), len(deltas)) + # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8f5a741e28a19..04eb5eceffbff 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -582,12 +582,16 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue startValue = find_value(params[0].get_obj(), "start"); UniValue endValue = find_value(params[0].get_obj(), "end"); - if (!startValue.isNum() || !endValue.isNum()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Start and end values are expected to be block heights"); - } + int start = 0; + int end = 0; - int start = startValue.get_int(); - int end = startValue.get_int(); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = endValue.get_int(); + if (end < start) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "End value is expected to be greater than start"); + } + } std::vector > addresses; @@ -598,8 +602,14 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 5fa85bc9ae38528ee7b69f272f3bc3ea83c90599 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 13:08:52 -0400 Subject: [PATCH 43/53] rpc: update oksafemode for address commands --- src/rpcserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 6717592fa9e4b..eb3060c51d96e 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,7 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, - { "blockchain", "getspentinfo", &getspentinfo, true }, + { "blockchain", "getspentinfo", &getspentinfo, false }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, @@ -316,7 +316,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ - { "addressindex", "getaddressmempool", &getaddressmempool, false }, + { "addressindex", "getaddressmempool", &getaddressmempool, true }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, From 9c5b709c6f5093e602d5a69a58301e5903d73b18 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 9 Feb 2016 12:37:05 +0100 Subject: [PATCH 44/53] tests: Make proxy_test work on travis servers without IPv6 Github-Pull: #7489 Rebased-From: 7539f1aae3b41279dc5d49e09f448a78a071e114 Cherry-picked-From: 9ca957bcd401de69c4c03904b9ee8b8b41052905 --- qa/rpc-tests/proxy_test.py | 74 +++++++++++++++----------- qa/rpc-tests/test_framework/netutil.py | 15 ++++++ 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py index 7f77e664d266c..b3c65573ea332 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/qa/rpc-tests/proxy_test.py @@ -7,6 +7,7 @@ from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.netutil import test_ipv6_local ''' Test plan: - Start bitcoind's with different proxy configurations @@ -34,6 +35,7 @@ class ProxyTest(BitcoinTestFramework): def __init__(self): + self.have_ipv6 = test_ipv6_local() # Create two proxies on different ports # ... one unauthenticated self.conf1 = Socks5Configuration() @@ -45,29 +47,36 @@ def __init__(self): self.conf2.addr = ('127.0.0.1', 14000 + (os.getpid() % 1000)) self.conf2.unauth = True self.conf2.auth = True - # ... one on IPv6 with similar configuration - self.conf3 = Socks5Configuration() - self.conf3.af = socket.AF_INET6 - self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) - self.conf3.unauth = True - self.conf3.auth = True + if self.have_ipv6: + # ... one on IPv6 with similar configuration + self.conf3 = Socks5Configuration() + self.conf3.af = socket.AF_INET6 + self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) + self.conf3.unauth = True + self.conf3.auth = True + else: + print "Warning: testing without local IPv6 support" self.serv1 = Socks5Server(self.conf1) self.serv1.start() self.serv2 = Socks5Server(self.conf2) self.serv2.start() - self.serv3 = Socks5Server(self.conf3) - self.serv3.start() + if self.have_ipv6: + self.serv3 = Socks5Server(self.conf3) + self.serv3.start() def setup_nodes(self): # Note: proxies are not used to connect to local nodes # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost - return start_nodes(4, self.options.tmpdir, extra_args=[ + args = [ ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], - ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - ]) + [] + ] + if self.have_ipv6: + args[3] = ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] + return start_nodes(4, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] @@ -84,18 +93,19 @@ def node_test(self, node, proxies, auth, test_onion=True): assert_equal(cmd.password, None) rv.append(cmd) - # Test: outgoing IPv6 connection through node - node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") - cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) - # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 - assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") - assert_equal(cmd.port, 5443) - if not auth: - assert_equal(cmd.username, None) - assert_equal(cmd.password, None) - rv.append(cmd) + if self.have_ipv6: + # Test: outgoing IPv6 connection through node + node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") + cmd = proxies[1].queue.get() + assert(isinstance(cmd, Socks5Command)) + # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") + assert_equal(cmd.port, 5443) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) if test_onion: # Test: outgoing onion connection through node @@ -135,10 +145,11 @@ def run_test(self): rv = self.node_test(self.nodes[2], [self.serv2, self.serv2, self.serv2, self.serv2], True) # Check that credentials as used for -proxyrandomize connections are unique credentials = set((x.username,x.password) for x in rv) - assert_equal(len(credentials), 4) + assert_equal(len(credentials), len(rv)) - # proxy on IPv6 localhost - self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False, False) + if self.have_ipv6: + # proxy on IPv6 localhost + self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False, False) def networks_dict(d): r = {} @@ -167,11 +178,12 @@ def networks_dict(d): assert_equal(n2[net]['proxy_randomize_credentials'], True) assert_equal(n2['onion']['reachable'], True) - n3 = networks_dict(self.nodes[3].getnetworkinfo()) - for net in ['ipv4','ipv6']: - assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) - assert_equal(n3[net]['proxy_randomize_credentials'], False) - assert_equal(n3['onion']['reachable'], False) + if self.have_ipv6: + n3 = networks_dict(self.nodes[3].getnetworkinfo()) + for net in ['ipv4','ipv6']: + assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) + assert_equal(n3[net]['proxy_randomize_credentials'], False) + assert_equal(n3['onion']['reachable'], False) if __name__ == '__main__': ProxyTest().main() diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index 50daa8793732a..bfdef76ad1caf 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -137,3 +137,18 @@ def addr_to_hex(addr): else: raise ValueError('Could not parse address %s' % addr) return binascii.hexlify(bytearray(addr)) + +def test_ipv6_local(): + ''' + Check for (local) IPv6 support. + ''' + import socket + # By using SOCK_DGRAM this will not actually make a connection, but it will + # fail if there is no route to IPv6 localhost. + have_ipv6 = True + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.connect(('::1', 0)) + except socket.error: + have_ipv6 = False + return have_ipv6 From 128c5e123213c77b313f1384775339c72819aedc Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 9 Feb 2016 22:17:09 +0000 Subject: [PATCH 45/53] Workaround Travis-side CI issues Github-Pull: #7487 Rebased-From: 149641e8fc9996da01eb76ffe4578828c40d37b5 c01f08db127883ff985889214eebdbe9513c729a 5d1148cb79856ac4695a0c7ac1cd28ada04eff34 1ecbb3b0f717c277f3db1a923fff16f7fc39432c Cherry-pick-From: 00d57b4d3a9a8e0a6e59ac639229debe14385ceb --- depends/builders/darwin.mk | 2 +- depends/builders/linux.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 200d6ed22a230..cedbddc57847e 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -7,7 +7,7 @@ build_darwin_OTOOL: = $(shell xcrun -f otool) build_darwin_NM: = $(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) build_darwin_SHA256SUM = shasum -a 256 -build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o +build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o #darwin host on darwin builder. overrides darwin host preferences. darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) diff --git a/depends/builders/linux.mk b/depends/builders/linux.mk index b03f42401047a..d6a304e4b4b9d 100644 --- a/depends/builders/linux.mk +++ b/depends/builders/linux.mk @@ -1,2 +1,2 @@ build_linux_SHA256SUM = sha256sum -build_linux_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o +build_linux_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o From 6c44620e5af7f51167271513917f087f193da243 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 20 Apr 2016 22:36:55 +0200 Subject: [PATCH 46/53] travis: switch to Trusty Github-Pull: #7920 Rebased-From: 06fdffd222ba0a00add4abe9fab9ad2c3e220d8f, 9267a47d86d0673eae9e504ee566aa4e0410d923, cf77fcdb1fe525b63b004ef729173f04bdb48882, 174023c9b008fc02316bce972b0c1031de3feee3, a33b7c9cb545985771d074748c0e368ca2d06702 Cherry-pick-From: 564aaa2cd0ee057d9eedc8da1c900d90f43c89c6 --- .travis.yml | 17 +++++++++-------- depends/packages/qt.mk | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1a5731f3af07a..f2fb8a4fc2fcd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,7 @@ # IPv6 support sudo: required -dist: precise -group: legacy +dist: trusty os: linux language: cpp @@ -37,22 +36,25 @@ matrix: - compiler: ": ARM" env: HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Win32" - env: HOST=i686-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-i686 g++-mingw-w64-i686 binutils-mingw-w64-i686 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="nsis g++-mingw-w64-i686 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - compiler: ": 32-bit + dash" - env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python-zmq" PPA="ppa:chris-lea/zeromq" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" + env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" - compiler: ": Win64" - env: HOST=x86_64-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="nsis g++-mingw-w64-x86-64 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - compiler: ": bitcoind" - env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc python-zmq" PPA="ppa:chris-lea/zeromq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" + env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc python-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" - compiler: ": No wallet" env: HOST=x86_64-unknown-linux-gnu DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Cross-Mac" env: HOST=x86_64-apple-darwin11 PACKAGES="cmake libcap-dev libz-dev libbz2-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" exclude: - compiler: gcc +before_install: + - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") install: - - if [ -n "$PACKAGES" ]; then sudo rm -f /etc/apt/sources.list.d/travis_ci_zeromq3-source.list; fi + - if [ -n "$PACKAGES" ]; then sudo rm -f /etc/apt/sources.list.d/google-chrome.list; fi - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi + - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: @@ -66,7 +68,6 @@ script: - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then export CCACHE_READONLY=1; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - ./configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make distdir PACKAGE=bitcoin VERSION=$HOST diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 901b761fde142..b2ed21f9d4da0 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -31,7 +31,7 @@ $(package)_config_opts += -no-iconv $(package)_config_opts += -no-gif $(package)_config_opts += -no-freetype $(package)_config_opts += -no-nis -$(package)_config_opts += -no-pch +$(package)_config_opts += -pch $(package)_config_opts += -no-qml-debug $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests From eb82f39655b6a35d52a04ce6a0f42176e20b06c7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 17:17:08 -0400 Subject: [PATCH 47/53] rpcclient: add params to be parsed as JSON There was an issue where getblockhashes wouldn't work from bitcoin-cli as the two params would be strings instead of integers. This fixes that issue, and will parse the first param as JSON for other addressindex related rpc methods. --- src/rpcclient.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 04715802372f5..27a09c0fc4b6a 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -102,6 +102,14 @@ static const CRPCConvertParam vRPCConvertParams[] = { "prioritisetransaction", 2 }, { "setban", 2 }, { "setban", 3 }, + { "getblockhashes", 0 }, + { "getblockhashes", 1 }, + { "getspentinfo", 0}, + { "getaddresstxids", 0}, + { "getaddressbalance", 0}, + { "getaddressdeltas", 0}, + { "getaddressutxos", 0}, + { "getaddressmempool", 0}, }; class CRPCConvertTable From 3c74fff5521a0c8d8f68730a0f7dbffaf91b4b99 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 18:33:22 -0400 Subject: [PATCH 48/53] rpc: include help text for addressindex and related commands --- src/rpcblockchain.cpp | 6 ++- src/rpcmisc.cpp | 99 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 276e99400c916..70ab3a98eb746 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -285,10 +285,12 @@ UniValue getblockhashes(const UniValue& params, bool fHelp) "1. high (numeric, required) The newer block timestamp\n" "2. low (numeric, required) The older block timestamp\n" "\nResult:\n" - "[" + "[\n" " \"hash\" (string) The block hash\n" - "]" + "]\n" "\nExamples:\n" + + HelpExampleCli("getblockhashes", "1231614698 1231024505") + + HelpExampleRpc("getblockhashes", "1231614698, 1231024505") ); unsigned int high = params[0].get_int(); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 04eb5eceffbff..8dc8f7146594a 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -462,7 +462,30 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) throw runtime_error( "getaddressmempool\n" "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" - ); + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + "}\n" + "\nResult:\n" + "[\n" + " {\n" + " \"address\" (string) The base58check encoded address\n" + " \"txid\" (string) The related txid\n" + " \"index\" (number) The related input or output index\n" + " \"satoshis\" (number) The difference of satoshis\n" + " \"timestamp\" (number) The time the transaction entered the mempool (seconds)\n" + " \"prevtxid\" (string) The previous txid (if spending)\n" + " \"prevout\" (string) The previous transaction output index (if spending)\n" + " }\n" + "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddressmempool", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressmempool", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") + ); std::vector > addresses; @@ -510,6 +533,14 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) throw runtime_error( "getaddressutxos\n" "\nReturns all unspent outputs for an address (requires addressindex to be enabled).\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + "}\n" "\nResult\n" "[\n" " {\n" @@ -521,7 +552,10 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) " \"satoshis\" (number) The number of satoshis of the output\n" " }\n" "]\n" - ); + "\nExamples:\n" + + HelpExampleCli("getaddressutxos", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressutxos", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") + ); std::vector > addresses; @@ -566,7 +600,17 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) throw runtime_error( "getaddressdeltas\n" "\nReturns all changes for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + " \"start\" (number) The start block height\n" + " \"end\" (number) The end block height\n" + "}\n" + "\nResult:\n" "[\n" " {\n" " \"satoshis\" (number) The difference of satoshis\n" @@ -576,6 +620,9 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " \"address\" (string) The base58check encoded address\n" " }\n" "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddressdeltas", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressdeltas", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); @@ -638,12 +685,23 @@ UniValue getaddressbalance(const UniValue& params, bool fHelp) if (fHelp || params.size() != 1) throw runtime_error( "getaddressbalance\n" - "\nReturns the balance for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nReturns the balance for an address(es) (requires addressindex to be enabled).\n" + "\nArguments:\n" "{\n" - " \"balance\" (string) The current balance\n" - " ,...\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" "}\n" + "\nResult:\n" + "{\n" + " \"balance\" (string) The current balance in satoshis\n" + " \"received\" (string) The total number of satoshis received (including change)\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getaddressbalance", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressbalance", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); std::vector > addresses; @@ -683,12 +741,25 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) if (fHelp || params.size() != 1) throw runtime_error( "getaddresstxids\n" - "\nReturns the txids for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nReturns the txids for an address(es) (requires addressindex to be enabled).\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + " \"start\" (number) The start block height\n" + " \"end\" (number) The end block height\n" + "}\n" + "\nResult:\n" "[\n" " \"transactionid\" (string) The transaction id\n" " ,...\n" "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddresstxids", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddresstxids", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); std::vector > addresses; @@ -755,12 +826,20 @@ UniValue getspentinfo(const UniValue& params, bool fHelp) throw runtime_error( "getspentinfo\n" "\nReturns the txid and index where an output is spent.\n" - "\nResult\n" + "\nArguments:\n" + "{\n" + " \"txid\" (string) The hex string of the txid\n" + " \"index\" (number) The start block height\n" + "}\n" + "\nResult:\n" "{\n" " \"txid\" (string) The transaction id\n" " \"index\" (number) The spending input index\n" " ,...\n" "}\n" + "\nExamples:\n" + + HelpExampleCli("getspentinfo", "'{\"txid\": \"0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9\", \"index\": 0}'") + + HelpExampleRpc("getspentinfo", "{\"txid\": \"0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9\", \"index\": 0}") ); UniValue txidValue = find_value(params[0].get_obj(), "txid"); From 1c022b9fc2773be9dfdca3b58b5daac26e04a382 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 10:27:03 -0400 Subject: [PATCH 49/53] rpc: add blockindex to getaddressdeltas method for the purposes of secondary sorting by block order --- qa/rpc-tests/addressindex.py | 1 + src/rpcmisc.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a07d4de1e0a6b..47102a02c7f5f 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -177,6 +177,7 @@ def run_test(self): balance3 += delta["satoshis"] assert_equal(balance3, change_amount) assert_equal(deltas[0]["address"], address2) + assert_equal(deltas[0]["blockindex"], 1) # Check that entire range will be queried deltasAll = self.nodes[1].getaddressdeltas({"addresses": [address2]}) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8dc8f7146594a..328e9eb290626 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -672,6 +672,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("blockindex", (int)it->first.txindex)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("address", address)); result.push_back(delta); From 87dfd136948d9834319895dacbcf87e786a16818 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 12 May 2016 16:16:44 -0400 Subject: [PATCH 50/53] rpc: include satoshis in verbose raw transaction --- qa/rpc-tests/txindex.py | 73 +++++++++++++++++++++++++++++++++++++++ src/rpcrawtransaction.cpp | 1 + 2 files changed, 74 insertions(+) create mode 100755 qa/rpc-tests/txindex.py diff --git a/qa/rpc-tests/txindex.py b/qa/rpc-tests/txindex.py new file mode 100755 index 0000000000000..b139253b769ec --- /dev/null +++ b/qa/rpc-tests/txindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test txindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class TxIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-txindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-txindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-txindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + print "Testing transaction index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + # Check verbose raw transaction results + verbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + assert_equal(verbose["vout"][0]["valueSat"], 5000000000); + assert_equal(verbose["vout"][0]["value"], 50); + + print "Passed\n" + + +if __name__ == '__main__': + TxIndexTest().main() diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 6fbec480edaa5..1d32da4aa5fbc 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -88,6 +88,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) const CTxOut& txout = tx.vout[i]; UniValue out(UniValue::VOBJ); out.push_back(Pair("value", ValueFromAmount(txout.nValue))); + out.push_back(Pair("valueSat", txout.nValue)); out.push_back(Pair("n", (int64_t)i)); UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(txout.scriptPubKey, o, true); From 16d35eb228232ed53f87cee233d0c8c3a9ca39eb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 13 May 2016 11:43:01 -0400 Subject: [PATCH 51/53] main: add amount and address to spentindex value --- src/main.cpp | 49 +++++++++++++++++++++++-------------------------- src/main.h | 14 +++++++++++++- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d750477e92b10..83e48801d5fc4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2492,39 +2492,36 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin for (size_t j = 0; j < tx.vin.size(); j++) { const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + uint160 hashBytes; + int addressType; - if (fSpentIndex) { - // add the spent index to determine the txid and input that spent an output - spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + hashBytes = uint160(vector (prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22)); + addressType = 2; + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + hashBytes = uint160(vector (prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23)); + addressType = 1; + } else { + hashBytes.SetNull(); + addressType = 0; } - if (fAddressIndex) { - - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + if (fAddressIndex && addressType > 0) { + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(addressType, hashBytes, pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - - } else { - continue; - } + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(addressType, hashBytes, input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } + if (fSpentIndex) { + // add the spent index to determine the txid and input that spent an output + // and to find the amount and address from an input + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight, prevout.nValue, addressType, hashBytes))); + } } + } if (fStrictPayToScriptHash) diff --git a/src/main.h b/src/main.h index 6c685e68ff267..c343a264c5438 100644 --- a/src/main.h +++ b/src/main.h @@ -325,6 +325,9 @@ struct CSpentIndexValue { uint256 txid; unsigned int inputIndex; int blockHeight; + CAmount satoshis; + int addressType; + uint160 addressHash; ADD_SERIALIZE_METHODS; @@ -333,12 +336,18 @@ struct CSpentIndexValue { READWRITE(txid); READWRITE(inputIndex); READWRITE(blockHeight); + READWRITE(satoshis); + READWRITE(addressType); + READWRITE(addressHash); } - CSpentIndexValue(uint256 t, unsigned int i, int h) { + CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { txid = t; inputIndex = i; blockHeight = h; + satoshis = s; + addressType = type; + addressHash = a; } CSpentIndexValue() { @@ -349,6 +358,9 @@ struct CSpentIndexValue { txid.SetNull(); inputIndex = 0; blockHeight = 0; + satoshis = 0; + addressType = 0; + addressHash.SetNull(); } bool IsNull() const { From 4c7dc871d1061ecab8238b5fb9efc0622d143e6a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 13 May 2016 11:43:29 -0400 Subject: [PATCH 52/53] rpc: add input value and address to getrawtransaction if spentindex enabled --- qa/rpc-tests/spentindex.py | 29 +++++++++++++++++++++++++++++ src/rpcrawtransaction.cpp | 14 ++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 6669df462f8de..5d0ab58b6a94b 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -62,18 +62,47 @@ def run_test(self): self.nodes[0].generate(1) self.sync_all() + print "Testing getspentinfo method..." + # Check that the spentinfo works standalone info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) assert_equal(info["height"], 106) + print "Testing getrawtransaction method..." + # Check that verbose raw transaction includes spent info txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) + # Check that verbose raw transaction includes input values + txVerbose2 = self.nodes[3].getrawtransaction(txid, 1) + assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + + # Check that verbose raw transaction includes address values and input values + privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG]) + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(txid, 16), 0))] + tx2.vout = [CTxOut(amount, scriptPubKey2)] + tx.rehash() + self.nodes[0].importprivkey(privkey) + signed_tx2 = self.nodes[0].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + txid2 = self.nodes[0].sendrawtransaction(signed_tx2["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + txVerbose3 = self.nodes[3].getrawtransaction(txid2, 1) + assert_equal(txVerbose3["vin"][0]["address"], address2) + assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + print "Passed\n" diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 1d32da4aa5fbc..ee39fbefd6254 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -78,6 +78,20 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); in.push_back(Pair("scriptSig", o)); + + // Add address and value info if spentindex enabled + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); + if (GetSpentIndex(spentKey, spentInfo)) { + in.push_back(Pair("value", ValueFromAmount(spentInfo.satoshis))); + in.push_back(Pair("valueSat", spentInfo.satoshis)); + if (spentInfo.addressType == 1) { + in.push_back(Pair("address", CBitcoinAddress(CKeyID(spentInfo.addressHash)).ToString())); + } else if (spentInfo.addressType == 2) { + in.push_back(Pair("address", CBitcoinAddress(CScriptID(spentInfo.addressHash)).ToString())); + } + } + } in.push_back(Pair("sequence", (int64_t)txin.nSequence)); vin.push_back(in); From 55fa4798ebbef084b46b047b8d16b22d713a3102 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 14:23:01 -0400 Subject: [PATCH 53/53] main: spentindex for the mempool --- qa/rpc-tests/spentindex.py | 17 +++++-- src/Makefile.am | 1 + src/main.cpp | 10 ++++ src/main.h | 76 +---------------------------- src/spentindex.h | 98 ++++++++++++++++++++++++++++++++++++++ src/txmempool.cpp | 66 +++++++++++++++++++++++++ src/txmempool.h | 11 +++++ 7 files changed, 200 insertions(+), 79 deletions(-) create mode 100644 src/spentindex.h diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 5d0ab58b6a94b..c233ca019789d 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -95,13 +95,22 @@ def run_test(self): self.nodes[0].importprivkey(privkey) signed_tx2 = self.nodes[0].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) txid2 = self.nodes[0].sendrawtransaction(signed_tx2["hex"], True) + + # Check the mempool index + self.sync_all() + txVerbose3 = self.nodes[1].getrawtransaction(txid2, 1) + assert_equal(txVerbose3["vin"][0]["address"], address2) + assert_equal(txVerbose3["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose3["vin"][0]["valueSat"], amount) + + # Check the database index self.nodes[0].generate(1) self.sync_all() - txVerbose3 = self.nodes[3].getrawtransaction(txid2, 1) - assert_equal(txVerbose3["vin"][0]["address"], address2) - assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) - assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + txVerbose4 = self.nodes[3].getrawtransaction(txid2, 1) + assert_equal(txVerbose4["vin"][0]["address"], address2) + assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose4["vin"][0]["valueSat"], amount) print "Passed\n" diff --git a/src/Makefile.am b/src/Makefile.am index 9632d65676ca9..c52371bea91cd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -80,6 +80,7 @@ endif # bitcoin core # BITCOIN_CORE_H = \ addressindex.h \ + spentindex.h \ addrman.h \ alert.h \ amount.h \ diff --git a/src/main.cpp b/src/main.cpp index 83e48801d5fc4..f384df004c71d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1415,10 +1415,17 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState &state, const C // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + + // Add memory address index if (fAddressIndex) { pool.addAddressIndex(entry, view); } + // Add memory spent index + if (fSpentIndex) { + pool.addSpentIndex(entry, view); + } + // trim mempool and check if tx was trimmed if (!fOverrideMempoolLimit) { LimitMempoolSize(pool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); @@ -1460,6 +1467,9 @@ bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) if (!fSpentIndex) return false; + if (mempool.getSpentIndex(key, value)) + return true; + if (!pblocktree->ReadSpentIndex(key, value)) return error("unable to get spent info"); diff --git a/src/main.h b/src/main.h index c343a264c5438..3f8a3fb32a219 100644 --- a/src/main.h +++ b/src/main.h @@ -17,6 +17,7 @@ #include "script/script_error.h" #include "sync.h" #include "versionbits.h" +#include "spentindex.h" #include #include @@ -293,81 +294,6 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; -struct CSpentIndexKey { - uint256 txid; - unsigned int outputIndex; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(txid); - READWRITE(outputIndex); - } - - CSpentIndexKey(uint256 t, unsigned int i) { - txid = t; - outputIndex = i; - } - - CSpentIndexKey() { - SetNull(); - } - - void SetNull() { - txid.SetNull(); - outputIndex = 0; - } - -}; - -struct CSpentIndexValue { - uint256 txid; - unsigned int inputIndex; - int blockHeight; - CAmount satoshis; - int addressType; - uint160 addressHash; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(txid); - READWRITE(inputIndex); - READWRITE(blockHeight); - READWRITE(satoshis); - READWRITE(addressType); - READWRITE(addressHash); - } - - CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { - txid = t; - inputIndex = i; - blockHeight = h; - satoshis = s; - addressType = type; - addressHash = a; - } - - CSpentIndexValue() { - SetNull(); - } - - void SetNull() { - txid.SetNull(); - inputIndex = 0; - blockHeight = 0; - satoshis = 0; - addressType = 0; - addressHash.SetNull(); - } - - bool IsNull() const { - return txid.IsNull(); - } -}; - struct CTimestampIndexIteratorKey { unsigned int timestamp; diff --git a/src/spentindex.h b/src/spentindex.h new file mode 100644 index 0000000000000..bd5da45d60dbc --- /dev/null +++ b/src/spentindex.h @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SPENTINDEX_H +#define BITCOIN_SPENTINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + CAmount satoshis; + int addressType; + uint160 addressHash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + READWRITE(satoshis); + READWRITE(addressType); + READWRITE(addressHash); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { + txid = t; + inputIndex = i; + blockHeight = h; + satoshis = s; + addressType = type; + addressHash = a; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + satoshis = 0; + addressType = 0; + addressHash.SetNull(); + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + +struct CSpentIndexKeyCompare +{ + bool operator()(const CSpentIndexKey& a, const CSpentIndexKey& b) const { + if (a.txid == b.txid) { + return a.outputIndex < b.outputIndex; + } else { + return a.txid < b.txid; + } + } +}; + +#endif // BITCOIN_SPENTINDEX_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index adbe662aba022..6ed4edbfd5cc9 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -496,6 +496,71 @@ bool CTxMemPool::removeAddressIndex(const uint256 txhash) return true; } +void CTxMemPool::addSpentIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + uint160 addressHash; + int addressType; + + if (prevout.scriptPubKey.IsPayToScriptHash()) { + addressHash = uint160(vector (prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22)); + addressType = 2; + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + addressHash = uint160(vector (prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23)); + addressType = 1; + } else { + addressHash.SetNull(); + addressType = 0; + } + + CSpentIndexKey key = CSpentIndexKey(input.prevout.hash, input.prevout.n); + CSpentIndexValue value = CSpentIndexValue(txhash, j, -1, prevout.nValue, addressType, addressHash); + + mapSpent.insert(make_pair(key, value)); + inserted.push_back(key); + + } + + mapSpentInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + LOCK(cs); + mapSpentIndex::iterator it; + + it = mapSpent.find(key); + if (it != mapSpent.end()) { + value = it->second; + return true; + } + return false; +} + +bool CTxMemPool::removeSpentIndex(const uint256 txhash) +{ + LOCK(cs); + mapSpentIndexInserted::iterator it = mapSpentInserted.find(txhash); + + if (it != mapSpentInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapSpent.erase(*mit); + } + mapSpentInserted.erase(it); + } + + return true; +} + void CTxMemPool::removeUnchecked(txiter it) { const uint256 hash = it->GetTx().GetHash(); @@ -510,6 +575,7 @@ void CTxMemPool::removeUnchecked(txiter it) nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); removeAddressIndex(hash); + removeSpentIndex(hash); } // Calculates descendants of entry that are not already in setDescendants, and adds to diff --git a/src/txmempool.h b/src/txmempool.h index 57091a02e62da..a94ff607bed89 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -10,6 +10,7 @@ #include #include "addressindex.h" +#include "spentindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -426,6 +427,12 @@ class CTxMemPool typedef std::map > addressDeltaMapInserted; addressDeltaMapInserted mapAddressInserted; + typedef std::map mapSpentIndex; + mapSpentIndex mapSpent; + + typedef std::map > mapSpentIndexInserted; + mapSpentIndexInserted mapSpentInserted; + void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); @@ -462,6 +469,10 @@ class CTxMemPool std::vector > &results); bool removeAddressIndex(const uint256 txhash); + void addSpentIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool removeSpentIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed);