From 6c48397cf37b4fae0a0809737406d315eb834b34 Mon Sep 17 00:00:00 2001 From: DenioD <41270280+DenioD@users.noreply.github.com> Date: Sat, 15 Feb 2020 14:14:39 +0100 Subject: [PATCH] Hush witness rework, many thanks to @CryptoForge for this awesome code https://github.com/Cryptoforge-alt/pirate/tree/pirate_witness_rework --- src/Makefile.am | 2 + src/init.cpp | 46 +- ...asyncrpcoperation_saplingconsolidation.cpp | 221 ++++ .../asyncrpcoperation_saplingconsolidation.h | 37 + src/wallet/db.cpp | 51 +- src/wallet/db.h | 1 + src/wallet/gtest/test_wallet.cpp | 44 +- src/wallet/wallet.cpp | 962 ++++++++++++++---- src/wallet/wallet.h | 59 +- src/wallet/walletdb.cpp | 6 + src/wallet/walletdb.h | 1 + 11 files changed, 1191 insertions(+), 239 deletions(-) create mode 100644 src/wallet/asyncrpcoperation_saplingconsolidation.cpp create mode 100644 src/wallet/asyncrpcoperation_saplingconsolidation.h diff --git a/src/Makefile.am b/src/Makefile.am index 2a304c5a5..d2fb7b174 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -248,6 +248,7 @@ BITCOIN_CORE_H = \ validationinterface.h \ version.h \ wallet/asyncrpcoperation_mergetoaddress.h \ + wallet/asyncrpcoperation_saplingconsolidation.h \ wallet/asyncrpcoperation_sendmany.h \ wallet/asyncrpcoperation_shieldcoinbase.h \ wallet/crypter.h \ @@ -368,6 +369,7 @@ libbitcoin_wallet_a_SOURCES = \ zcbenchmarks.cpp \ zcbenchmarks.h \ wallet/asyncrpcoperation_mergetoaddress.cpp \ + wallet/asyncrpcoperation_saplingconsolidation.cpp \ wallet/asyncrpcoperation_sendmany.cpp \ wallet/asyncrpcoperation_shieldcoinbase.cpp \ wallet/crypter.cpp \ diff --git a/src/init.cpp b/src/init.cpp index 36ed5e0cb..6428bf8dc 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -57,6 +57,7 @@ #ifdef ENABLE_WALLET #include "wallet/wallet.h" #include "wallet/walletdb.h" +#include "wallet/asyncrpcoperation_saplingconsolidation.h" #endif #include @@ -449,6 +450,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Wallet options:")); strUsage += HelpMessageOpt("-disablewallet", _("Do not load the wallet and disable wallet RPC calls")); strUsage += HelpMessageOpt("-keypool=", strprintf(_("Set key pool size to (default: %u)"), 100)); + strUsage += HelpMessageOpt("-consolidation", _("Enable auto Sapling note consolidation")); + strUsage += HelpMessageOpt("-consolidatesaplingaddress=", _("Specify Sapling Address to Consolidate. (default: all)")); + strUsage += HelpMessageOpt("-consolidationtxfee", strprintf(_("Fee amount in Puposhis used send consolidation transactions. (default %i)"), DEFAULT_CONSOLIDATION_FEE)); + strUsage += HelpMessageOpt("-deletetx", _("Enable Old Transaction Deletion")); + strUsage += HelpMessageOpt("-deleteinterval", strprintf(_("Delete transaction every blocks during inital block download (default: %i)"), DEFAULT_TX_DELETE_INTERVAL)); + strUsage += HelpMessageOpt("-keeptxnum", strprintf(_("Keep the last transactions (default: %i)"), DEFAULT_TX_RETENTION_LASTTX)); + strUsage += HelpMessageOpt("-keeptxfornblocks", strprintf(_("Keep transactions for at least blocks (default: %i)"), DEFAULT_TX_RETENTION_BLOCKS)); if (showDebug) strUsage += HelpMessageOpt("-mintxfee=", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(CWallet::minTxFee.GetFeePerK()))); @@ -500,7 +508,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0)); strUsage += HelpMessageOpt("-nuparams=hexBranchId:activationHeight", "Use given activation height for specified network upgrade (regtest-only)"); } - string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, http, libevent, lock, mempool, net, partitioncheck, pow, proxy, prune, " + string debugCategories = "addrman, alert, bench, coindb, db, deletetx, estimatefee, http, libevent, lock, mempool, net, partitioncheck, pow, proxy, prune, " "rand, reindex, rpc, selectcoins, tor, zmq, zrpc, zrpcunsafe (implies zrpc)"; // Don't translate these strUsage += HelpMessageOpt("-debug=", strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + _("If is not supplied or if = 1, output all debugging information.") + " " + _(" can be:") + " " + debugCategories + "."); @@ -1953,6 +1961,42 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) pwalletMain->GenerateNewSeed(); } + //Set Sapling Consolidation + pwalletMain->fSaplingConsolidationEnabled = GetBoolArg("-consolidation", false); + fConsolidationTxFee = GetArg("-consolidationtxfee", DEFAULT_CONSOLIDATION_FEE); + fConsolidationMapUsed = !mapMultiArgs["-consolidatesaplingaddress"].empty(); + + //Validate Sapling Addresses + vector& vaddresses = mapMultiArgs["-consolidatesaplingaddress"]; + for (int i = 0; i < vaddresses.size(); i++) { + LogPrintf("Consolidating Sapling Address: %s\n", vaddresses[i]); + auto zAddress = DecodePaymentAddress(vaddresses[i]); + if (!IsValidPaymentAddress(zAddress)) { + return InitError("Invalid consolidation address"); + } + } + + //Set Transaction Deletion Options + fTxDeleteEnabled = GetBoolArg("-deletetx", false); + fTxConflictDeleteEnabled = GetBoolArg("-deleteconflicttx", true); + + fDeleteInterval = GetArg("-deleteinterval", DEFAULT_TX_DELETE_INTERVAL); + if (fDeleteInterval < 1) + return InitError("deleteinterval must be greater than 0"); + + fKeepLastNTransactions = GetArg("-keeptxnum", DEFAULT_TX_RETENTION_LASTTX); + if (fKeepLastNTransactions < 1) + return InitError("keeptxnum must be greater than 0"); + + fDeleteTransactionsAfterNBlocks = GetArg("-keeptxfornblocks", DEFAULT_TX_RETENTION_BLOCKS); + if (fDeleteTransactionsAfterNBlocks < 1) + return InitError("keeptxfornblocks must be greater than 0"); + + if (fDeleteTransactionsAfterNBlocks < MAX_REORG_LENGTH + 1 ) { + LogPrintf("keeptxfornblock is less the MAX_REORG_LENGTH, Setting to %i\n", MAX_REORG_LENGTH + 1); + fDeleteTransactionsAfterNBlocks = MAX_REORG_LENGTH + 1; + } + if (fFirstRun) { // Create new keyUser and set as default key diff --git a/src/wallet/asyncrpcoperation_saplingconsolidation.cpp b/src/wallet/asyncrpcoperation_saplingconsolidation.cpp new file mode 100644 index 000000000..9152f0d8c --- /dev/null +++ b/src/wallet/asyncrpcoperation_saplingconsolidation.cpp @@ -0,0 +1,221 @@ +#include "assert.h" +#include "boost/variant/static_visitor.hpp" +#include "asyncrpcoperation_saplingconsolidation.h" +#include "init.h" +#include "key_io.h" +#include "rpc/protocol.h" +#include "random.h" +#include "sync.h" +#include "tinyformat.h" +#include "transaction_builder.h" +#include "util.h" +#include "utilmoneystr.h" +#include "wallet.h" + +CAmount fConsolidationTxFee = DEFAULT_CONSOLIDATION_FEE; +bool fConsolidationMapUsed = false; +const int CONSOLIDATION_EXPIRY_DELTA = 15; + + +AsyncRPCOperation_saplingconsolidation::AsyncRPCOperation_saplingconsolidation(int targetHeight) : targetHeight_(targetHeight) {} + +AsyncRPCOperation_saplingconsolidation::~AsyncRPCOperation_saplingconsolidation() {} + +void AsyncRPCOperation_saplingconsolidation::main() { + if (isCancelled()) + return; + + set_state(OperationStatus::EXECUTING); + start_execution_clock(); + + bool success = false; + + try { + success = main_impl(); + } catch (const UniValue& objError) { + int code = find_value(objError, "code").get_int(); + std::string message = find_value(objError, "message").get_str(); + set_error_code(code); + set_error_message(message); + } catch (const runtime_error& e) { + set_error_code(-1); + set_error_message("runtime error: " + string(e.what())); + } catch (const logic_error& e) { + set_error_code(-1); + set_error_message("logic error: " + string(e.what())); + } catch (const exception& e) { + set_error_code(-1); + set_error_message("general exception: " + string(e.what())); + } catch (...) { + set_error_code(-2); + set_error_message("unknown error"); + } + + stop_execution_clock(); + + if (success) { + set_state(OperationStatus::SUCCESS); + } else { + set_state(OperationStatus::FAILED); + } + + std::string s = strprintf("%s: Sapling Consolidation transaction created. (status=%s", getId(), getStateAsString()); + if (success) { + s += strprintf(", success)\n"); + } else { + s += strprintf(", error=%s)\n", getErrorMessage()); + } + + LogPrintf("%s", s); +} + +bool AsyncRPCOperation_saplingconsolidation::main_impl() { + LogPrint("zrpcunsafe", "%s: Beginning AsyncRPCOperation_saplingconsolidation.\n", getId()); + auto consensusParams = Params().GetConsensus(); + auto nextActivationHeight = NextActivationHeight(targetHeight_, consensusParams); + if (nextActivationHeight && targetHeight_ + CONSOLIDATION_EXPIRY_DELTA >= nextActivationHeight.get()) { + LogPrint("zrpcunsafe", "%s: Consolidation txs would be created before a NU activation but may expire after. Skipping this round.\n", getId()); + setConsolidationResult(0, 0, std::vector()); + return true; + } + + std::vector sproutEntries; + std::vector saplingEntries; + std::set addresses; + { + LOCK2(cs_main, pwalletMain->cs_wallet); + // We set minDepth to 11 to avoid unconfirmed notes and in anticipation of specifying + // an anchor at height N-10 for each Sprout JoinSplit description + // Consider, should notes be sorted? + pwalletMain->GetFilteredNotes(sproutEntries, saplingEntries, "", 11); + if (fConsolidationMapUsed) { + const vector& v = mapMultiArgs["-consolidatesaplingaddress"]; + for(int i = 0; i < v.size(); i++) { + auto zAddress = DecodePaymentAddress(v[i]); + if (boost::get(&zAddress) != nullptr) { + libzcash::SaplingPaymentAddress saplingAddress = boost::get(zAddress); + addresses.insert(saplingAddress ); + } + } + } else { + pwalletMain->GetSaplingPaymentAddresses(addresses); + } + } + + int numTxCreated = 0; + std::vector consolidationTxIds; + CAmount amountConsolidated = 0; + CCoinsViewCache coinsView(pcoinsTip); + + for (auto addr : addresses) { + libzcash::SaplingExtendedSpendingKey extsk; + if (pwalletMain->GetSaplingExtendedSpendingKey(addr, extsk)) { + + std::vector fromNotes; + CAmount amountToSend = 0; + int maxQuantity = rand() % 35 + 10; + for (const SaplingNoteEntry& saplingEntry : saplingEntries) { + + libzcash::SaplingIncomingViewingKey ivk; + pwalletMain->GetSaplingIncomingViewingKey(boost::get(saplingEntry.address), ivk); + + //Select Notes from that same address we will be sending to. + if (ivk == extsk.expsk.full_viewing_key().in_viewing_key()) { + amountToSend += CAmount(saplingEntry.note.value()); + fromNotes.push_back(saplingEntry); + } + + //Only use a randomly determined number of notes between 10 and 45 + if (fromNotes.size() >= maxQuantity) + break; + + } + + //random minimum 2 - 12 required + int minQuantity = rand() % 10 + 2; + if (fromNotes.size() < minQuantity) + continue; + + amountConsolidated += amountToSend; + auto builder = TransactionBuilder(consensusParams, targetHeight_, pwalletMain); + //builder.SetExpiryHeight(targetHeight_ + CONSOLIDATION_EXPIRY_DELTA); + LogPrint("zrpcunsafe", "%s: Beginning creating transaction with Sapling output amount=%s\n", getId(), FormatMoney(amountToSend - fConsolidationTxFee)); + + // Select Sapling notes + std::vector ops; + std::vector notes; + for (auto fromNote : fromNotes) { + ops.push_back(fromNote.op); + notes.push_back(fromNote.note); + } + + // Fetch Sapling anchor and witnesses + uint256 anchor; + std::vector> witnesses; + { + LOCK2(cs_main, pwalletMain->cs_wallet); + pwalletMain->GetSaplingNoteWitnesses(ops, witnesses, anchor); + } + + // Add Sapling spends + for (size_t i = 0; i < notes.size(); i++) { + if (!witnesses[i]) { + LogPrint("zrpcunsafe", "%s: Missing Witnesses. Stopping.\n", getId()); + break; + } + builder.AddSaplingSpend(extsk.expsk, notes[i], anchor, witnesses[i].get()); + } + + builder.SetFee(fConsolidationTxFee); + builder.AddSaplingOutput(extsk.expsk.ovk, addr, amountToSend - fConsolidationTxFee); + //CTransaction tx = builder.Build(); + + auto maybe_tx = builder.Build(); + if (!maybe_tx) { + LogPrint("zrpcunsafe", "%s: Failed to build transaction.", getId()); + break; + } + CTransaction tx = maybe_tx.get(); + + if (isCancelled()) { + LogPrint("zrpcunsafe", "%s: Canceled. Stopping.\n", getId()); + break; + } + + pwalletMain->CommitConsolidationTx(tx); + LogPrint("zrpcunsafe", "%s: Committed consolidation transaction with txid=%s\n", getId(), tx.GetHash().ToString()); + amountConsolidated += amountToSend - fConsolidationTxFee; + consolidationTxIds.push_back(tx.GetHash().ToString()); + + } + } + + LogPrint("zrpcunsafe", "%s: Created %d transactions with total Sapling output amount=%s\n", getId(), numTxCreated, FormatMoney(amountConsolidated)); + setConsolidationResult(numTxCreated, amountConsolidated, consolidationTxIds); + return true; + +} + +void AsyncRPCOperation_saplingconsolidation::setConsolidationResult(int numTxCreated, const CAmount& amountConsolidated, const std::vector& consolidationTxIds) { + UniValue res(UniValue::VOBJ); + res.push_back(Pair("num_tx_created", numTxCreated)); + res.push_back(Pair("amount_consolidated", FormatMoney(amountConsolidated))); + UniValue txIds(UniValue::VARR); + for (const std::string& txId : consolidationTxIds) { + txIds.push_back(txId); + } + res.push_back(Pair("consolidation_txids", txIds)); + set_result(res); +} + +void AsyncRPCOperation_saplingconsolidation::cancel() { + set_state(OperationStatus::CANCELLED); +} + +UniValue AsyncRPCOperation_saplingconsolidation::getStatus() const { + UniValue v = AsyncRPCOperation::getStatus(); + UniValue obj = v.get_obj(); + obj.push_back(Pair("method", "saplingconsolidation")); + obj.push_back(Pair("target_height", targetHeight_)); + return obj; +} diff --git a/src/wallet/asyncrpcoperation_saplingconsolidation.h b/src/wallet/asyncrpcoperation_saplingconsolidation.h new file mode 100644 index 000000000..8e38204c7 --- /dev/null +++ b/src/wallet/asyncrpcoperation_saplingconsolidation.h @@ -0,0 +1,37 @@ +#include "amount.h" +#include "asyncrpcoperation.h" +#include "univalue.h" +#include "zcash/Address.hpp" +#include "zcash/zip32.h" + +//Default fee used for consolidation transactions +static const CAmount DEFAULT_CONSOLIDATION_FEE = 0; +extern CAmount fConsolidationTxFee; +extern bool fConsolidationMapUsed; + +class AsyncRPCOperation_saplingconsolidation : public AsyncRPCOperation +{ +public: + AsyncRPCOperation_saplingconsolidation(int targetHeight); + virtual ~AsyncRPCOperation_saplingconsolidation(); + + // We don't want to be copied or moved around + AsyncRPCOperation_saplingconsolidation(AsyncRPCOperation_saplingconsolidation const&) = delete; // Copy construct + AsyncRPCOperation_saplingconsolidation(AsyncRPCOperation_saplingconsolidation&&) = delete; // Move construct + AsyncRPCOperation_saplingconsolidation& operator=(AsyncRPCOperation_saplingconsolidation const&) = delete; // Copy assign + AsyncRPCOperation_saplingconsolidation& operator=(AsyncRPCOperation_saplingconsolidation&&) = delete; // Move assign + + virtual void main(); + + virtual void cancel(); + + virtual UniValue getStatus() const; + +private: + int targetHeight_; + + bool main_impl(); + + void setConsolidationResult(int numTxCreated, const CAmount& amountConsolidated, const std::vector& consolidationTxIds); + +}; \ No newline at end of file diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index dd0880b75..1814e9209 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -105,7 +105,7 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) nEnvFlags |= DB_PRIVATE; dbenv->set_lg_dir(pathLogDir.string().c_str()); - dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet + dbenv->set_cachesize(1, 0x100000, 1); // 1 MiB should be enough for just the wallet, Increased by 1 GB dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); @@ -181,6 +181,55 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFu return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } +bool CDBEnv::Compact(const std::string& strFile) +{ + LOCK(cs_db); + + DB_COMPACT dbcompact; + dbcompact.compact_fillpercent = 80; + dbcompact.compact_pages = DB_MAX_PAGES; + dbcompact.compact_timeout = 0; + + DB_COMPACT *pdbcompact; + pdbcompact = &dbcompact; + + int result = 1; + if (mapDb[strFile] != NULL) { + Db* pdb = mapDb[strFile]; + result = pdb->compact(NULL, NULL, NULL, pdbcompact, DB_FREE_SPACE, NULL); + delete pdb; + mapDb[strFile] = NULL; + + switch (result) + { + case DB_LOCK_DEADLOCK: + LogPrint("db","Deadlock %i\n", result); + break; + case DB_LOCK_NOTGRANTED: + LogPrint("db","Lock Not Granted %i\n", result); + break; + case DB_REP_HANDLE_DEAD: + LogPrint("db","Handle Dead %i\n", result); + break; + case DB_REP_LOCKOUT: + LogPrint("db","Rep Lockout %i\n", result); + break; + case EACCES: + LogPrint("db","Eacces %i\n", result); + break; + case EINVAL: + LogPrint("db","Error Invalid %i\n", result); + break; + case 0: + LogPrint("db","Wallet Compact Sucessful\n"); + break; + default: + LogPrint("db","Compact result int %i\n", result); + } + } + return (result == 0); +} + bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector& vResult) { LOCK(cs_db); diff --git a/src/wallet/db.h b/src/wallet/db.h index 8ad246de4..31ba88f11 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -88,6 +88,7 @@ public: * NOTE: reads the entire database into memory, so cannot be used * for huge databases. */ + bool Compact(const std::string& strFile); typedef std::pair, std::vector > KeyValPair; bool Salvage(const std::string& strFile, bool fAggressive, std::vector& vResult); diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index 676239d0b..dcc7fdfdd 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -53,12 +53,10 @@ public: return CCryptoKeyStore::Unlock(vMasterKeyIn); } - void IncrementNoteWitnesses(const CBlockIndex* pindex, - const CBlock* pblock, - SproutMerkleTree& sproutTree, - SaplingMerkleTree& saplingTree) { - CWallet::IncrementNoteWitnesses(pindex, pblock, sproutTree, saplingTree); + void BuildWitnessCache(const CBlockIndex* pindex, bool witnessOnly) { + CWallet::BuildWitnessCache(pindex, witnessOnly); } + void DecrementNoteWitnesses(const CBlockIndex* pindex) { CWallet::DecrementNoteWitnesses(pindex); } @@ -116,7 +114,7 @@ std::pair CreateValidBlock(TestWallet& wallet, wallet.AddToWallet(wtx, true, NULL); block.vtx.push_back(wtx); - wallet.IncrementNoteWitnesses(&index, &block, sproutTree, saplingTree); + wallet.BuildWitnessCache(&index, false); return std::make_pair(jsoutpt, saplingNotes[0]); } @@ -724,8 +722,8 @@ TEST(WalletTests, GetConflictedSaplingNotes) { wallet.AddToWallet(wtx, true, NULL); // Simulate receiving new block and ChainTip signal - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + wallet.BuildWitnessCache(&fakeIndex, false); + wallet.UpdateNullifierNoteMapForBlock(&block); // Retrieve the updated wtx from wallet uint256 hash = wtx.GetHash(); @@ -1008,8 +1006,8 @@ TEST(WalletTests, NavigateFromSaplingNullifierToNote) { } // Simulate receiving new block and ChainTip signal - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + wallet.BuildWitnessCache(&fakeIndex, false); + wallet.UpdateNullifierNoteMapForBlock(&block); // Retrieve the updated wtx from wallet uint256 hash = wtx.GetHash(); @@ -1126,8 +1124,8 @@ TEST(WalletTests, SpentSaplingNoteIsFromMe) { // Simulate receiving new block and ChainTip signal. // This triggers calculation of nullifiers for notes belonging to this wallet // in the output descriptions of wtx. - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + wallet.BuildWitnessCache(&fakeIndex, false); + wallet.UpdateNullifierNoteMapForBlock(&block); // Retrieve the updated wtx from wallet wtx = wallet.mapWallet[wtx.GetHash()]; @@ -1263,7 +1261,7 @@ TEST(WalletTests, CachedWitnessesEmptyChain) { CBlockIndex index(block); SproutMerkleTree sproutTree; SaplingMerkleTree saplingTree; - wallet.IncrementNoteWitnesses(&index, &block, sproutTree, saplingTree); + wallet.BuildWitnessCache(&index, false); ::GetWitnessesAndAnchors(wallet, sproutNotes, saplingNotes, sproutWitnesses, saplingWitnesses); @@ -1354,7 +1352,7 @@ TEST(WalletTests, CachedWitnessesChainTip) { EXPECT_NE(anchors1.second, anchors3.second); // Re-incrementing with the same block should give the same result - wallet.IncrementNoteWitnesses(&index2, &block2, sproutTree, saplingTree); + wallet.BuildWitnessCache(&index2, false); auto anchors4 = GetWitnessesAndAnchors(wallet, sproutNotes, saplingNotes, sproutWitnesses, saplingWitnesses); EXPECT_NE(anchors4.first, anchors4.second); @@ -1364,7 +1362,7 @@ TEST(WalletTests, CachedWitnessesChainTip) { EXPECT_EQ(anchors2.second, anchors4.second); // Incrementing with the same block again should not change the cache - wallet.IncrementNoteWitnesses(&index2, &block2, sproutTree, saplingTree); + wallet.BuildWitnessCache(&index2, false); std::vector> sproutWitnesses5; std::vector> saplingWitnesses5; @@ -1447,7 +1445,7 @@ TEST(WalletTests, CachedWitnessesDecrementFirst) { EXPECT_NE(anchors2.second, anchors4.second); // Re-incrementing with the same block should give the same result - wallet.IncrementNoteWitnesses(&index2, &block2, sproutTree, saplingTree); + wallet.BuildWitnessCache(&index2, false); auto anchors5 = GetWitnessesAndAnchors(wallet, sproutNotes, saplingNotes, sproutWitnesses, saplingWitnesses); @@ -1504,7 +1502,7 @@ TEST(WalletTests, CachedWitnessesCleanIndex) { for (size_t i = 0; i < numBlocks; i++) { SproutMerkleTree sproutRiPrevTree {sproutRiTree}; SaplingMerkleTree saplingRiPrevTree {saplingRiTree}; - wallet.IncrementNoteWitnesses(&(indices[i]), &(blocks[i]), sproutRiTree, saplingRiTree); + wallet.BuildWitnessCache(&indices[i], false); auto anchors = GetWitnessesAndAnchors(wallet, sproutNotes, saplingNotes, sproutWitnesses, saplingWitnesses); for (size_t j = 0; j < numBlocks; j++) { @@ -1531,7 +1529,7 @@ TEST(WalletTests, CachedWitnessesCleanIndex) { } { - wallet.IncrementNoteWitnesses(&(indices[i]), &(blocks[i]), sproutRiPrevTree, saplingRiPrevTree); + wallet.BuildWitnessCache(&indices[i], false); auto anchors = GetWitnessesAndAnchors(wallet, sproutNotes, saplingNotes, sproutWitnesses, saplingWitnesses); for (size_t j = 0; j < numBlocks; j++) { EXPECT_TRUE((bool) sproutWitnesses[j]); @@ -1886,8 +1884,8 @@ TEST(WalletTests, UpdatedSaplingNoteData) { wallet.AddToWallet(wtx, true, NULL); // Simulate receiving new block and ChainTip signal - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + wallet.BuildWitnessCache(&fakeIndex, false); + wallet.UpdateNullifierNoteMapForBlock(&block); // Retrieve the updated wtx from wallet uint256 hash = wtx.GetHash(); @@ -1915,7 +1913,7 @@ TEST(WalletTests, UpdatedSaplingNoteData) { EXPECT_EQ(2, wtx2.mapSaplingNoteData.size()); EXPECT_EQ(1, wtx2.mapSaplingNoteData[sop0].witnesses.size()); // wtx2 has fake witness for payment output - EXPECT_EQ(0, wtx2.mapSaplingNoteData[sop1].witnesses.size()); // wtx2 never had incrementnotewitness called + EXPECT_EQ(0, wtx2.mapSaplingNoteData[sop1].witnesses.size()); // wtx2 never had BuildWitnessCache called // After updating, they should be the same EXPECT_TRUE(wallet.UpdatedNoteData(wtx2, wtx)); @@ -2040,8 +2038,8 @@ TEST(WalletTests, MarkAffectedSaplingTransactionsDirty) { wallet.AddToWallet(wtx, true, NULL); // Simulate receiving new block and ChainTip signal - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + wallet.BuildWitnessCache(&fakeIndex, false); + wallet.UpdateNullifierNoteMapForBlock(&block); // Retrieve the updated wtx from wallet uint256 hash = wtx.GetHash(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b52ccdb9f..6e73f2a1a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -20,6 +20,7 @@ ******************************************************************************/ #include "wallet/wallet.h" +#include "asyncrpcqueue.h" #include "checkpoints.h" #include "coincontrol.h" @@ -38,6 +39,7 @@ #include "zcash/Note.hpp" #include "crypter.h" #include "coins.h" +#include "wallet/asyncrpcoperation_saplingconsolidation.h" #include "zcash/zip32.h" #include "cc/CCinclude.h" @@ -65,6 +67,11 @@ CBlockIndex *komodo_chainactive(int32_t height); extern std::string DONATION_PUBKEY; int32_t komodo_dpowconfs(int32_t height,int32_t numconfs); int tx_height( const uint256 &hash ); +bool fTxDeleteEnabled = false; +bool fTxConflictDeleteEnabled = false; +int fDeleteInterval = DEFAULT_TX_DELETE_INTERVAL; +unsigned int fDeleteTransactionsAfterNBlocks = DEFAULT_TX_RETENTION_BLOCKS; +unsigned int fKeepLastNTransactions = DEFAULT_TX_RETENTION_LASTTX; /** * Fees smaller than this (in satoshi) are considered zero fee (for transaction creation) @@ -584,11 +591,56 @@ void CWallet::ChainTip(const CBlockIndex *pindex, bool added) { if (added) { - IncrementNoteWitnesses(pindex, pblock, sproutTree, saplingTree); + // Prevent witness cache building && consolidation transactions + // from being created when node is syncing after launch, + // and also when node wakes up from suspension/hibernation and incoming blocks are old. + bool initialDownloadCheck = IsInitialBlockDownload(); + if (!initialDownloadCheck && + pblock->GetBlockTime() > GetAdjustedTime() - 8640) //Last 144 blocks 2.4 * 60 * 60 + { + BuildWitnessCache(pindex, false); + RunSaplingConsolidation(pindex->GetHeight()); + DeleteWalletTransactions(pindex); + } else { + //Build intial witnesses on every block + BuildWitnessCache(pindex, true); + if (initialDownloadCheck && pindex->GetHeight() % fDeleteInterval == 0) { + DeleteWalletTransactions(pindex); + } + } } else { DecrementNoteWitnesses(pindex); + UpdateNullifierNoteMapForBlock(pblock); + } +} + +void CWallet::RunSaplingConsolidation(int blockHeight) { + if (!NetworkUpgradeActive(blockHeight, Params().GetConsensus(), Consensus::UPGRADE_SAPLING)) { + return; + } + LOCK(cs_wallet); + if (!fSaplingConsolidationEnabled) { + return; } - UpdateSaplingNullifierNoteMapForBlock(pblock); + + int consolidateInterval = rand() % 5 + 5; + if (blockHeight % consolidateInterval == 0) { + std::shared_ptr q = getAsyncRPCQueue(); + std::shared_ptr lastOperation = q->getOperationForId(saplingConsolidationOperationId); + if (lastOperation != nullptr) { + lastOperation->cancel(); + } + pendingSaplingConsolidationTxs.clear(); + std::shared_ptr operation(new AsyncRPCOperation_saplingconsolidation(blockHeight + 5)); + saplingConsolidationOperationId = operation->getId(); + q->addOperation(operation); + } +} + +void CWallet::CommitConsolidationTx(const CTransaction& tx) { + CWalletTx wtx(this, tx); + CReserveKey reservekey(pwalletMain); + CommitTransaction(wtx, reservekey); } void CWallet::SetBestChain(const CBlockLocator& loc) @@ -872,6 +924,22 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const return false; } +unsigned int CWallet::GetSpendDepth(const uint256& hash, unsigned int n) const +{ + const COutPoint outpoint(hash, n); + pair range; + range = mapTxSpends.equal_range(outpoint); + + for (TxSpends::const_iterator it = range.first; it != range.second; ++it) + { + const uint256& wtxid = it->second; + std::map::const_iterator mit = mapWallet.find(wtxid); + if (mit != mapWallet.end() && mit->second.GetDepthInMainChain() >= 0) + return mit->second.GetDepthInMainChain(); // Spent + } + return 0; +} + /** * Note is spent if any non-conflicted transaction * spends it: @@ -890,6 +958,20 @@ bool CWallet::IsSproutSpent(const uint256& nullifier) const { return false; } +unsigned int CWallet::GetSproutSpendDepth(const uint256& nullifier) const { + pair range; + range = mapTxSproutNullifiers.equal_range(nullifier); + + for (TxNullifiers::const_iterator it = range.first; it != range.second; ++it) { + const uint256& wtxid = it->second; + std::map::const_iterator mit = mapWallet.find(wtxid); + if (mit != mapWallet.end() && mit->second.GetDepthInMainChain() >= 0) { + return mit->second.GetDepthInMainChain(); // Spent + } + } + return 0; +} + bool CWallet::IsSaplingSpent(const uint256& nullifier) const { pair range; range = mapTxSaplingNullifiers.equal_range(nullifier); @@ -904,6 +986,20 @@ bool CWallet::IsSaplingSpent(const uint256& nullifier) const { return false; } +unsigned int CWallet::GetSaplingSpendDepth(const uint256& nullifier) const { + pair range; + range = mapTxSaplingNullifiers.equal_range(nullifier); + + for (TxNullifiers::const_iterator it = range.first; it != range.second; ++it) { + const uint256& wtxid = it->second; + std::map::const_iterator mit = mapWallet.find(wtxid); + if (mit != mapWallet.end() && mit->second.GetDepthInMainChain() >= 0) { + return mit->second.GetDepthInMainChain(); // Spent + } + } + return 0; +} + void CWallet::AddToTransparentSpends(const COutPoint& outpoint, const uint256& wtxid) { mapTxSpends.insert(make_pair(outpoint, wtxid)); @@ -992,241 +1088,369 @@ void CWallet::ClearNoteWitnessCache() item.second.witnessHeight = -1; } } - nWitnessCacheSize = 0; - //fprintf(stderr,"Clear witness cache\n"); } -template -void CopyPreviousWitnesses(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize) +void CWallet::DecrementNoteWitnesses(const CBlockIndex* pindex) { - for (auto& item : noteDataMap) { - auto* nd = &(item.second); - // Only increment witnesses that are behind the current height - if (nd->witnessHeight < indexHeight) { - // Check the validity of the cache - // The only time a note witnessed above the current height - // would be invalid here is during a reindex when blocks - // have been decremented, and we are incrementing the blocks - // immediately after. - assert(nWitnessCacheSize >= nd->witnesses.size()); - // Witnesses being incremented should always be either -1 - // (never incremented or decremented) or one below indexHeight - assert((nd->witnessHeight == -1) || (nd->witnessHeight == indexHeight - 1)); - // Copy the witness for the previous block if we have one - if (nd->witnesses.size() > 0) { - nd->witnesses.push_front(nd->witnesses.front()); + LOCK(cs_wallet); + + extern int32_t KOMODO_REWIND; + + for (std::pair& wtxItem : mapWallet) { + //Sprout + for (auto& item : wtxItem.second.mapSproutNoteData) { + auto* nd = &(item.second); + if (nd->nullifier && pwalletMain->GetSproutSpendDepth(*item.second.nullifier) <= WITNESS_CACHE_SIZE) { + // Only decrement witnesses that are not above the current height + if (nd->witnessHeight <= pindex->GetHeight()) { + if (nd->witnesses.size() > 1) { + // indexHeight is the height of the block being removed, so + // the new witness cache height is one below it. + nd->witnesses.pop_front(); + nd->witnessHeight = pindex->GetHeight() - 1; + } + } } - if (nd->witnesses.size() > WITNESS_CACHE_SIZE) { - nd->witnesses.pop_back(); + } + //Sapling + for (auto& item : wtxItem.second.mapSaplingNoteData) { + auto* nd = &(item.second); + if (nd->nullifier && pwalletMain->GetSaplingSpendDepth(*item.second.nullifier) <= WITNESS_CACHE_SIZE) { + // Only decrement witnesses that are not above the current height + if (nd->witnessHeight <= pindex->GetHeight()) { + if (nd->witnesses.size() > 1) { + // indexHeight is the height of the block being removed, so + // the new witness cache height is one below it. + nd->witnesses.pop_front(); + nd->witnessHeight = pindex->GetHeight() - 1; + } + } } } } + assert(KOMODO_REWIND != 0 || WITNESS_CACHE_SIZE != _COINBASE_MATURITY+10); } -template -void AppendNoteCommitment(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const uint256& note_commitment) +template +void ClearSingleNoteWitnessCache(NoteData* nd) { - for (auto& item : noteDataMap) { - auto* nd = &(item.second); - if (nd->witnessHeight < indexHeight && nd->witnesses.size() > 0) { - // Check the validity of the cache - // See comment in CopyPreviousWitnesses about validity. - assert(nWitnessCacheSize >= nd->witnesses.size()); - nd->witnesses.front().append(note_commitment); - } - } + nd->witnesses.clear(); + nd->witnessHeight = -1; + nd->witnessRootValidated = false; } -template -void WitnessNoteIfMine(std::map& noteDataMap, int indexHeight, int64_t nWitnessCacheSize, const OutPoint& key, const Witness& witness) +int CWallet::SproutWitnessMinimumHeight(const uint256& nullifier, int nWitnessHeight, int nMinimumHeight) { - if (noteDataMap.count(key) && noteDataMap[key].witnessHeight < indexHeight) { - auto* nd = &(noteDataMap[key]); - if (nd->witnesses.size() > 0) { - // We think this can happen because we write out the - // witness cache state after every block increment or - // decrement, but the block index itself is written in - // batches. So if the node crashes in between these two - // operations, it is possible for IncrementNoteWitnesses - // to be called again on previously-cached blocks. This - // doesn't affect existing cached notes because of the - // NoteData::witnessHeight checks. See #1378 for details. - LogPrintf("Inconsistent witness cache state found for %s\n- Cache size: %d\n- Top (height %d): %s\n- New (height %d): %s\n", - key.ToString(), nd->witnesses.size(), - nd->witnessHeight, - nd->witnesses.front().root().GetHex(), - indexHeight, - witness.root().GetHex()); - nd->witnesses.clear(); - } - nd->witnesses.push_front(witness); - // Set height to one less than pindex so it gets incremented - nd->witnessHeight = indexHeight - 1; - // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); + if (GetSproutSpendDepth(nullifier) <= WITNESS_CACHE_SIZE) { + nMinimumHeight = min(nWitnessHeight, nMinimumHeight); } + return nMinimumHeight; } - -template -void UpdateWitnessHeights(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize) +int CWallet::SaplingWitnessMinimumHeight(const uint256& nullifier, int nWitnessHeight, int nMinimumHeight) { - for (auto& item : noteDataMap) { - auto* nd = &(item.second); - if (nd->witnessHeight < indexHeight) { - nd->witnessHeight = indexHeight; - // Check the validity of the cache - // See comment in CopyPreviousWitnesses about validity. - assert(nWitnessCacheSize >= nd->witnesses.size()); - } + if (GetSaplingSpendDepth(nullifier) <= WITNESS_CACHE_SIZE) { + nMinimumHeight = min(nWitnessHeight, nMinimumHeight); } + return nMinimumHeight; } -void CWallet::IncrementNoteWitnesses(const CBlockIndex* pindex, - const CBlock* pblockIn, - SproutMerkleTree& sproutTree, - SaplingMerkleTree& saplingTree) +int CWallet::VerifyAndSetInitialWitness(const CBlockIndex* pindex, bool witnessOnly) { - LOCK(cs_wallet); - for (std::pair& wtxItem : mapWallet) { - ::CopyPreviousWitnesses(wtxItem.second.mapSproutNoteData, pindex->GetHeight(), nWitnessCacheSize); - ::CopyPreviousWitnesses(wtxItem.second.mapSaplingNoteData, pindex->GetHeight(), nWitnessCacheSize); - } + LOCK2(cs_main, cs_wallet); - if (nWitnessCacheSize < WITNESS_CACHE_SIZE) { - nWitnessCacheSize += 1; - } + int nWitnessTxIncrement = 0; + int nWitnessTotalTxCount = mapWallet.size(); + int nMinimumHeight = pindex->GetHeight(); - const CBlock* pblock {pblockIn}; - CBlock block; - if (!pblock) { - ReadBlockFromDisk(block, pindex, false); + for (std::pair& wtxItem : mapWallet) { + nWitnessTxIncrement += 1; + + if (wtxItem.second.mapSproutNoteData.empty() && wtxItem.second.mapSaplingNoteData.empty()) + continue; + + if (wtxItem.second.GetDepthInMainChain() > 0) { + auto wtxHash = wtxItem.second.GetHash(); + int wtxHeight = mapBlockIndex[wtxItem.second.hashBlock]->GetHeight(); + + for (mapSproutNoteData_t::value_type& item : wtxItem.second.mapSproutNoteData) { + + auto op = item.first; + auto* nd = &(item.second); + CBlockIndex* pblockindex; + uint256 blockRoot; + uint256 witnessRoot; + + if (!nd->nullifier) + ::ClearSingleNoteWitnessCache(nd); + + if (!nd->witnesses.empty() && nd->witnessHeight > 0) { + + //Skip all functions for validated witness while witness only = true + if (nd->witnessRootValidated && witnessOnly) + continue; + + //Skip Validation when witness root has been validated + if (nd->witnessRootValidated) { + nMinimumHeight = SproutWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + + //Skip Validation when witness height is greater that block height + if (nd->witnessHeight > pindex->GetHeight() - 1) { + nMinimumHeight = SproutWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + + //Validate the witness at the witness height + witnessRoot = nd->witnesses.front().root(); + pblockindex = chainActive[nd->witnessHeight]; + blockRoot = pblockindex->hashFinalSproutRoot; + if (witnessRoot == blockRoot) { + nd->witnessRootValidated = true; + nMinimumHeight = SproutWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + } + + //Clear witness Cache for all other scenarios + pblockindex = chainActive[wtxHeight]; + ::ClearSingleNoteWitnessCache(nd); + + LogPrintf("Setting Inital Sprout Witness for tx %s, %i of %i\n", wtxHash.ToString(), nWitnessTxIncrement, nWitnessTotalTxCount); + + SproutMerkleTree sproutTree; + blockRoot = pblockindex->pprev->hashFinalSproutRoot; + pcoinsTip->GetSproutAnchorAt(blockRoot, sproutTree); + + //Cycle through blocks and transactions building sprout tree until the commitment needed is reached + const CBlock* pblock; + CBlock block; + ReadBlockFromDisk(block, pblockindex, 1); pblock = █ - } - for (const CTransaction& tx : pblock->vtx) { - auto hash = tx.GetHash(); - bool txIsOurs = mapWallet.count(hash); - // Sprout - for (size_t i = 0; i < tx.vjoinsplit.size(); i++) { + for (const CTransaction& tx : block.vtx) { + auto hash = tx.GetHash(); + + for (size_t i = 0; i < tx.vjoinsplit.size(); i++) { const JSDescription& jsdesc = tx.vjoinsplit[i]; for (uint8_t j = 0; j < jsdesc.commitments.size(); j++) { - const uint256& note_commitment = jsdesc.commitments[j]; - sproutTree.append(note_commitment); + const uint256& note_commitment = jsdesc.commitments[j]; - // Increment existing witnesses - for (std::pair& wtxItem : mapWallet) { - ::AppendNoteCommitment(wtxItem.second.mapSproutNoteData, pindex->GetHeight(), nWitnessCacheSize, note_commitment); - } + // Increment existing witness until the end of the block + if (!nd->witnesses.empty()) { + nd->witnesses.front().append(note_commitment); + } + + //Only needed for intial witness + if (nd->witnesses.empty()) { + sproutTree.append(note_commitment); // If this is our note, witness it - if (txIsOurs) { - JSOutPoint jsoutpt {hash, i, j}; - ::WitnessNoteIfMine(mapWallet[hash].mapSproutNoteData, pindex->GetHeight(), nWitnessCacheSize, jsoutpt, sproutTree.witness()); + if (hash == wtxHash) { + JSOutPoint outPoint {hash, i, j}; + if (op == outPoint) { + nd->witnesses.push_front(sproutTree.witness()); + } } + } } + } } - // Sapling - for (uint32_t i = 0; i < tx.vShieldedOutput.size(); i++) { + + nd->witnessHeight = pblockindex->GetHeight(); + UpdateSproutNullifierNoteMapWithTx(wtxItem.second); + nMinimumHeight = SproutWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + } + + for (mapSaplingNoteData_t::value_type& item : wtxItem.second.mapSaplingNoteData) { + + auto op = item.first; + auto* nd = &(item.second); + CBlockIndex* pblockindex; + uint256 blockRoot; + uint256 witnessRoot; + + if (!nd->nullifier) + ::ClearSingleNoteWitnessCache(nd); + + if (!nd->witnesses.empty() && nd->witnessHeight > 0) { + + //Skip all functions for validated witness while witness only = true + if (nd->witnessRootValidated && witnessOnly) + continue; + + //Skip Validation when witness root has been validated + if (nd->witnessRootValidated) { + nMinimumHeight = SaplingWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + + //Skip Validation when witness height is greater that block height + if (nd->witnessHeight > pindex->GetHeight() - 1) { + nMinimumHeight = SaplingWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + + //Validate the witness at the witness height + witnessRoot = nd->witnesses.front().root(); + pblockindex = chainActive[nd->witnessHeight]; + blockRoot = pblockindex->hashFinalSaplingRoot; + if (witnessRoot == blockRoot) { + nd->witnessRootValidated = true; + nMinimumHeight = SaplingWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + continue; + } + } + + //Clear witness Cache for all other scenarios + pblockindex = chainActive[wtxHeight]; + ::ClearSingleNoteWitnessCache(nd); + + LogPrintf("Setting Inital Sapling Witness for tx %s, %i of %i\n", wtxHash.ToString(), nWitnessTxIncrement, nWitnessTotalTxCount); + + SaplingMerkleTree saplingTree; + blockRoot = pblockindex->pprev->hashFinalSaplingRoot; + pcoinsTip->GetSaplingAnchorAt(blockRoot, saplingTree); + + //Cycle through blocks and transactions building sapling tree until the commitment needed is reached + const CBlock* pblock; + CBlock block; + ReadBlockFromDisk(block, pblockindex, 1); + pblock = █ + + for (const CTransaction& tx : block.vtx) { + auto hash = tx.GetHash(); + + // Sapling + for (uint32_t i = 0; i < tx.vShieldedOutput.size(); i++) { const uint256& note_commitment = tx.vShieldedOutput[i].cm; - saplingTree.append(note_commitment); - // Increment existing witnesses - for (std::pair& wtxItem : mapWallet) { - ::AppendNoteCommitment(wtxItem.second.mapSaplingNoteData, pindex->GetHeight(), nWitnessCacheSize, note_commitment); + // Increment existing witness until the end of the block + if (!nd->witnesses.empty()) { + nd->witnesses.front().append(note_commitment); } - // If this is our note, witness it - if (txIsOurs) { + //Only needed for intial witness + if (nd->witnesses.empty()) { + saplingTree.append(note_commitment); + + // If this is our note, witness it + if (hash == wtxHash) { SaplingOutPoint outPoint {hash, i}; - ::WitnessNoteIfMine(mapWallet[hash].mapSaplingNoteData, pindex->GetHeight(), nWitnessCacheSize, outPoint, saplingTree.witness()); + if (op == outPoint) { + nd->witnesses.push_front(saplingTree.witness()); + } + } } + } } + nd->witnessHeight = pblockindex->GetHeight(); + UpdateSaplingNullifierNoteMapWithTx(wtxItem.second); + nMinimumHeight = SaplingWitnessMinimumHeight(*item.second.nullifier, nd->witnessHeight, nMinimumHeight); + } } + } - // Update witness heights - for (std::pair& wtxItem : mapWallet) { - ::UpdateWitnessHeights(wtxItem.second.mapSproutNoteData, pindex->GetHeight(), nWitnessCacheSize); - ::UpdateWitnessHeights(wtxItem.second.mapSaplingNoteData, pindex->GetHeight(), nWitnessCacheSize); - } - - // For performance reasons, we write out the witness cache in - // CWallet::SetBestChain() (which also ensures that overall consistency - // of the wallet.dat is maintained). + return nMinimumHeight; } -template -bool DecrementNoteWitnesses(NoteDataMap& noteDataMap, int indexHeight, int64_t nWitnessCacheSize) +void CWallet::BuildWitnessCache(const CBlockIndex* pindex, bool witnessOnly) { - extern int32_t KOMODO_REWIND; - for (auto& item : noteDataMap) { - auto* nd = &(item.second); - // Only decrement witnesses that are not above the current height - if (nd->witnessHeight <= indexHeight) { - // Check the validity of the cache - // See comment below (this would be invalid if there were a - // prior decrement). - assert(nWitnessCacheSize >= nd->witnesses.size()); - // Witnesses being decremented should always be either -1 - // (never incremented or decremented) or equal to the height - // of the block being removed (indexHeight) - if (!((nd->witnessHeight == -1) || (nd->witnessHeight == indexHeight))) - { - printf("at height %d\n", indexHeight); - return false; + LOCK2(cs_main, cs_wallet); + + int startHeight = VerifyAndSetInitialWitness(pindex, witnessOnly) + 1; + + if (startHeight > pindex->GetHeight() || witnessOnly) { + return; + } + + uint256 sproutRoot; + uint256 saplingRoot; + CBlockIndex* pblockindex = chainActive[startHeight]; + int height = chainActive.Height(); + + while (pblockindex) { + + if (pblockindex->GetHeight() % 100 == 0 && pblockindex->GetHeight() < height - 5) { + LogPrintf("Building Witnesses for block %i %.4f complete\n", pblockindex->GetHeight(), pblockindex->GetHeight() / double(height)); + } + + SproutMerkleTree sproutTree; + sproutRoot = pblockindex->pprev->hashFinalSproutRoot; + pcoinsTip->GetSproutAnchorAt(sproutRoot, sproutTree); + + SaplingMerkleTree saplingTree; + saplingRoot = pblockindex->pprev->hashFinalSaplingRoot; + pcoinsTip->GetSaplingAnchorAt(saplingRoot, saplingTree); + + //Cycle through blocks and transactions building sapling tree until the commitment needed is reached + CBlock block; + ReadBlockFromDisk(block, pblockindex, 1); + + for (std::pair& wtxItem : mapWallet) { + + if (wtxItem.second.mapSproutNoteData.empty() && wtxItem.second.mapSaplingNoteData.empty()) + continue; + + if (wtxItem.second.GetDepthInMainChain() > 0) { + + //Sprout + for (mapSproutNoteData_t::value_type& item : wtxItem.second.mapSproutNoteData) { + auto* nd = &(item.second); + if (nd->nullifier && nd->witnessHeight == pblockindex->GetHeight() - 1 + && GetSproutSpendDepth(*item.second.nullifier) <= WITNESS_CACHE_SIZE) { + + + nd->witnesses.push_front(nd->witnesses.front()); + while (nd->witnesses.size() > WITNESS_CACHE_SIZE) { + nd->witnesses.pop_back(); } - if (nd->witnesses.size() > 0) { - nd->witnesses.pop_front(); + + for (const CTransaction& tx : block.vtx) { + for (size_t i = 0; i < tx.vjoinsplit.size(); i++) { + const JSDescription& jsdesc = tx.vjoinsplit[i]; + for (uint8_t j = 0; j < jsdesc.commitments.size(); j++) { + const uint256& note_commitment = jsdesc.commitments[j]; + nd->witnesses.front().append(note_commitment); + } + } } - // indexHeight is the height of the block being removed, so - // the new witness cache height is one below it. - nd->witnessHeight = indexHeight - 1; - } - // Check the validity of the cache - // Technically if there are notes witnessed above the current - // height, their cache will now be invalid (relative to the new - // value of nWitnessCacheSize). However, this would only occur - // during a reindex, and by the time the reindex reaches the tip - // of the chain again, the existing witness caches will be valid - // again. - // We don't set nWitnessCacheSize to zero at the start of the - // reindex because the on-disk blocks had already resulted in a - // chain that didn't trigger the assertion below. - if (nd->witnessHeight < indexHeight) { - // Subtract 1 to compare to what nWitnessCacheSize will be after - // decrementing. - assert((nWitnessCacheSize - 1) >= nd->witnesses.size()); - } - } - assert(KOMODO_REWIND != 0 || nWitnessCacheSize > 0 || WITNESS_CACHE_SIZE != _COINBASE_MATURITY+10); - return true; -} + nd->witnessHeight = pblockindex->GetHeight(); + } + } + //Sapling + for (mapSaplingNoteData_t::value_type& item : wtxItem.second.mapSaplingNoteData) { + auto* nd = &(item.second); + if (nd->nullifier && nd->witnessHeight == pblockindex->GetHeight() - 1 + && GetSaplingSpendDepth(*item.second.nullifier) <= WITNESS_CACHE_SIZE) { -void CWallet::DecrementNoteWitnesses(const CBlockIndex* pindex) -{ - LOCK(cs_wallet); - for (std::pair& wtxItem : mapWallet) { - if (!::DecrementNoteWitnesses(wtxItem.second.mapSproutNoteData, pindex->GetHeight(), nWitnessCacheSize)) - needsRescan = true; - if (!::DecrementNoteWitnesses(wtxItem.second.mapSaplingNoteData, pindex->GetHeight(), nWitnessCacheSize)) - needsRescan = true; - } - if ( WITNESS_CACHE_SIZE == _COINBASE_MATURITY+10 ) - { - nWitnessCacheSize -= 1; - // TODO: If nWitnessCache is zero, we need to regenerate the caches (#1302) - assert(nWitnessCacheSize > 0); - } - else - { - if ( nWitnessCacheSize > 0 ) - nWitnessCacheSize--; + nd->witnesses.push_front(nd->witnesses.front()); + while (nd->witnesses.size() > WITNESS_CACHE_SIZE) { + nd->witnesses.pop_back(); + } + + for (const CTransaction& tx : block.vtx) { + for (uint32_t i = 0; i < tx.vShieldedOutput.size(); i++) { + const uint256& note_commitment = tx.vShieldedOutput[i].cm; + nd->witnesses.front().append(note_commitment); + } + } + nd->witnessHeight = pblockindex->GetHeight(); + } + } + } } - // For performance reasons, we write out the witness cache in - // CWallet::SetBestChain() (which also ensures that overall consistency - // of the wallet.dat is maintained). + + if (pblockindex == pindex) + break; + + pblockindex = chainActive.Next(pblockindex); + + } + } bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) @@ -1429,6 +1653,48 @@ void CWallet::UpdateNullifierNoteMapWithTx(const CWalletTx& wtx) } } +/** + * Update mapSproutNullifiersToNotes, computing the nullifier from a cached witness if necessary. + */ +void CWallet::UpdateSproutNullifierNoteMapWithTx(CWalletTx& wtx) { + LOCK(cs_wallet); + + ZCNoteDecryption dec; + for (mapSproutNoteData_t::value_type& item : wtx.mapSproutNoteData) { + SproutNoteData nd = item.second; + + if (nd.witnesses.empty()) { + // If there are no witnesses, erase the nullifier and associated mapping. + if (nd.nullifier) { + mapSproutNullifiersToNotes.erase(nd.nullifier.get()); + } + nd.nullifier = boost::none; + } + else { + if (GetNoteDecryptor(nd.address, dec)) { + auto i = item.first.js; + auto hSig = wtx.vjoinsplit[i].h_sig( + *pzcashParams, wtx.joinSplitPubKey); + auto optNullifier = GetSproutNoteNullifier( + wtx.vjoinsplit[i], + item.second.address, + dec, + hSig, + item.first.n); + + if (!optNullifier) { + // This should not happen. If it does, maybe the position has been corrupted or miscalculated? + assert(false); + } + + uint256 nullifier = optNullifier.get(); + mapSproutNullifiersToNotes[nullifier] = item.first; + item.second.nullifier = nullifier; + } + } + } +} + /** * Update mapSaplingNullifiersToNotes, computing the nullifier from a cached witness if necessary. */ @@ -1479,13 +1745,14 @@ void CWallet::UpdateSaplingNullifierNoteMapWithTx(CWalletTx& wtx) { * Iterate over transactions in a block and update the cached Sapling nullifiers * for transactions which belong to the wallet. */ -void CWallet::UpdateSaplingNullifierNoteMapForBlock(const CBlock *pblock) { +void CWallet::UpdateNullifierNoteMapForBlock(const CBlock *pblock) { LOCK(cs_wallet); for (const CTransaction& tx : pblock->vtx) { auto hash = tx.GetHash(); bool txIsOurs = mapWallet.count(hash); if (txIsOurs) { + UpdateSproutNullifierNoteMapWithTx(mapWallet[hash]); UpdateSaplingNullifierNoteMapWithTx(mapWallet[hash]); } } @@ -2725,6 +2992,298 @@ void CWallet::WitnessNoteCommitment(std::vector commitments, } } +/** + * Reorder the transactions based on block hieght and block index. + * Transactions can get out of order when they are deleted and subsequently + * re-added during intial load rescan. + */ + +void CWallet::ReorderWalletTransactions(std::map, CWalletTx> &mapSorted, int64_t &maxOrderPos) { + LOCK2(cs_main, cs_wallet); + + int maxSortNumber = chainActive.Tip()->GetHeight() + 1; + + for (map::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + { + CWalletTx wtx = it->second; + int confirms = wtx.GetDepthInMainChain(); + maxOrderPos = max(maxOrderPos, wtx.nOrderPos); + + if (confirms > 0) { + int wtxHeight = mapBlockIndex[wtx.hashBlock]->GetHeight(); + auto key = std::make_pair(wtxHeight, wtx.nIndex); + mapSorted.insert(make_pair(key, wtx)); + } + else { + auto key = std::make_pair(maxSortNumber, 0); + mapSorted.insert(std::make_pair(key, wtx)); + maxSortNumber++; + } + } +} + /**Update the nOrderPos with passed in ordered map. + */ + +void CWallet::UpdateWalletTransactionOrder(std::map, CWalletTx> &mapSorted, bool resetOrder) { + LOCK2(cs_main, cs_wallet); + + int64_t previousPosition = 0; + std::map mapUpdatedTxs; + + //Check the postion of each transaction relative to the previous one. + for (map, CWalletTx>::iterator it = mapSorted.begin(); it != mapSorted.end(); ++it) { + CWalletTx wtx = it->second; + const uint256 wtxid = wtx.GetHash(); + + if (wtx.nOrderPos <= previousPosition || resetOrder) { + previousPosition++; + wtx.nOrderPos = previousPosition; + mapUpdatedTxs.insert(std::make_pair(wtxid, wtx)); + } + else { + previousPosition = wtx.nOrderPos; + } + } + + //Update transactions nOrderPos for transactions that changed + CWalletDB walletdb(strWalletFile, "r+", false); + for (map::iterator it = mapUpdatedTxs.begin(); it != mapUpdatedTxs.end(); ++it) { + CWalletTx wtx = it->second; + LogPrint("deletetx","Reorder Tx - Updating Positon to %i for Tx %s\n ", wtx.nOrderPos, wtx.GetHash().ToString()); + wtx.WriteToDisk(&walletdb); + mapWallet[wtx.GetHash()].nOrderPos = wtx.nOrderPos; + } + + //Update Next Wallet Tx Positon + nOrderPosNext = previousPosition++; + CWalletDB(strWalletFile).WriteOrderPosNext(nOrderPosNext); + LogPrint("deletetx","Reorder Tx - Total Transactions Reordered %i, Next Position %i\n ", mapUpdatedTxs.size(), nOrderPosNext); + +} + +/** + * Delete transactions from the Wallet + */ +void CWallet::DeleteTransactions(std::vector &removeTxs) { + LOCK(cs_wallet); + + CWalletDB walletdb(strWalletFile, "r+", false); + + for (int i = 0; i< removeTxs.size(); i++) { + if (mapWallet.erase(removeTxs[i])) { + walletdb.EraseTx(removeTxs[i]); + LogPrint("deletetx","Delete Tx - Deleting tx %s, %i.\n", removeTxs[i].ToString(),i); + } else { + LogPrint("deletetx","Delete Tx - Deleting tx %failed.\n", removeTxs[i].ToString()); + return; + } + } +} + +void CWallet::DeleteWalletTransactions(const CBlockIndex* pindex) { + + LOCK2(cs_main, cs_wallet); + + int nDeleteAfter = (int)fDeleteTransactionsAfterNBlocks; + bool runCompact = false; + + if (pindex && fTxDeleteEnabled) { + + //Check for acentries - exit function if found + { + std::list acentries; + CWalletDB walletdb(strWalletFile); + walletdb.ListAccountCreditDebit("*", acentries); + if (acentries.size() > 0) { + LogPrintf("deletetx not compatible to account entries\n"); + return; + } + } + //delete transactions + + //Sort Transactions by block and block index + int64_t maxOrderPos = 0; + std::map, CWalletTx> mapSorted; + ReorderWalletTransactions(mapSorted, maxOrderPos); + if (maxOrderPos > int64_t(mapSorted.size())*10) { + //reset the postion when the max postion is 10x bigger than the + //number of transactions in the wallet + LogPrint("deletetx","Reorder Tx - maxOrderPos %i mapSorted Size %i\n", maxOrderPos, int64_t(mapSorted.size())*10); + UpdateWalletTransactionOrder(mapSorted, true); + } + else { + UpdateWalletTransactionOrder(mapSorted, false); + } + + //Process Transactions in sorted order + int txConflictCount = 0; + int txUnConfirmed = 0; + int txCount = 0; + int txSaveCount = 0; + std::vector removeTxs; + + for (auto & item : mapSorted) + { + + CWalletTx& wtx = item.second; + const uint256& wtxid = wtx.GetHash(); + bool deleteTx = true; + txCount += 1; + int wtxDepth = wtx.GetDepthInMainChain(); + + //Keep anything newer than N Blocks + if (wtxDepth == 0) + txUnConfirmed++; + + if (wtxDepth < nDeleteAfter && wtxDepth >= 0) { + LogPrint("deletetx","DeleteTx - Transaction above minimum depth, tx %s\n", wtx.GetHash().ToString()); + deleteTx = false; + txSaveCount++; + continue; + } else if (wtxDepth == -1) { + //Enabled by default + if (!fTxConflictDeleteEnabled) { + LogPrint("deletetx","DeleteTx - Conflict delete is not enabled tx %s\n", wtx.GetHash().ToString()); + deleteTx = false; + txSaveCount++; + continue; + } else { + txConflictCount++; + } + } else { + + //Check for unspent inputs or spend less than N Blocks ago. (Sapling) + for (auto & pair : wtx.mapSaplingNoteData) { + SaplingNoteData nd = pair.second; + if (!nd.nullifier || pwalletMain->GetSaplingSpendDepth(*nd.nullifier) <= fDeleteTransactionsAfterNBlocks) { + LogPrint("deletetx","DeleteTx - Unspent sapling input tx %s\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Check for outputs that no longer have parents in the wallet. Exclude parents that are in the same transaction. (Sapling) + for (int i = 0; i < wtx.vShieldedSpend.size(); i++) { + const SpendDescription& spendDesc = wtx.vShieldedSpend[i]; + if (pwalletMain->IsSaplingNullifierFromMe(spendDesc.nullifier)) { + const uint256& parentHash = pwalletMain->mapSaplingNullifiersToNotes[spendDesc.nullifier].hash; + const CWalletTx* parent = pwalletMain->GetWalletTx(parentHash); + if (parent != NULL && parentHash != wtxid) { + LogPrint("deletetx","DeleteTx - Parent of sapling tx %s found\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Check for unspent inputs or spend less than N Blocks ago. (Sprout) + for (auto & pair : wtx.mapSproutNoteData) { + SproutNoteData nd = pair.second; + if (!nd.nullifier || pwalletMain->GetSproutSpendDepth(*nd.nullifier) <= fDeleteTransactionsAfterNBlocks) { + LogPrint("deletetx","DeleteTx - Unspent sprout input tx %s\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Check for outputs that no longer have parents in the wallet. Exclude parents that are in the same transaction. (Sprout) + for (int i = 0; i < wtx.vjoinsplit.size(); i++) { + const JSDescription& jsdesc = wtx.vjoinsplit[i]; + for (const uint256 &nullifier : jsdesc.nullifiers) { + // JSOutPoint op = pwalletMain->mapSproutNullifiersToNotes[nullifier]; + if (pwalletMain->IsSproutNullifierFromMe(nullifier)) { + const uint256& parentHash = pwalletMain->mapSproutNullifiersToNotes[nullifier].hash; + const CWalletTx* parent = pwalletMain->GetWalletTx(parentHash); + if (parent != NULL && parentHash != wtxid) { + LogPrint("deletetx","DeleteTx - Parent of sprout tx %s found\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Check for unspent inputs or spend less than N Blocks ago. (Transparent) + for (unsigned int i = 0; i < wtx.vout.size(); i++) { + CTxDestination address; + ExtractDestination(wtx.vout[i].scriptPubKey, address); + if(IsMine(wtx.vout[i])) { + if (pwalletMain->GetSpendDepth(wtx.GetHash(), i) <= fDeleteTransactionsAfterNBlocks) { + LogPrint("deletetx","DeleteTx - Unspent transparent input tx %s\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Chcek for output with that no longer have parents in the wallet. (Transparent) + for (int i = 0; i < wtx.vin.size(); i++) { + const CTxIn& txin = wtx.vin[i]; + const uint256& parentHash = txin.prevout.hash; + const CWalletTx* parent = pwalletMain->GetWalletTx(txin.prevout.hash); + if (parent != NULL && parentHash != wtxid) { + LogPrint("deletetx","DeleteTx - Parent of transparent tx %s found\n", wtx.GetHash().ToString()); + deleteTx = false; + continue; + } + } + + if (!deleteTx) { + txSaveCount++; + continue; + } + + //Keep Last N Transactions + if (mapSorted.size() - txCount < fKeepLastNTransactions + txConflictCount + txUnConfirmed) { + LogPrint("deletetx","DeleteTx - Transaction set position %i, tx %s\n", mapSorted.size() - txCount, wtxid.ToString()); + deleteTx = false; + txSaveCount++; + continue; + } + } + + //Collect everything else for deletion + if (deleteTx && int(removeTxs.size()) < MAX_DELETE_TX_SIZE) { + removeTxs.push_back(wtxid); + runCompact = true; + } + } + + //Delete Transactions from wallet + DeleteTransactions(removeTxs); + LogPrintf("Delete Tx - Total Transaction Count %i, Transactions Deleted %i\n ", txCount, int(removeTxs.size())); + + //Compress Wallet + if (runCompact) + CWalletDB::Compact(bitdb,strWalletFile); + } +} + /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already @@ -2738,8 +3297,6 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) CBlockIndex* pindex = pindexStart; - std::vector myTxHashes; - { LOCK2(cs_main, cs_wallet); @@ -2761,7 +3318,6 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) BOOST_FOREACH(CTransaction& tx, block.vtx) { if (AddToWalletIfInvolvingMe(tx, &block, fUpdate)) { - myTxHashes.push_back(tx.GetHash()); ret++; } } @@ -2776,27 +3332,23 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) assert(pcoinsTip->GetSaplingAnchorAt(pindex->pprev->hashFinalSaplingRoot, saplingTree)); } } - // Increment note witness caches - ChainTip(pindex, &block, sproutTree, saplingTree, true); - pindex = chainActive.Next(pindex); + // Build inital witness caches + BuildWitnessCache(pindex, true); + + //Delete Transactions + if (pindex->GetHeight() % fDeleteInterval == 0) + DeleteWalletTransactions(pindex); + if (GetTime() >= nNow + 60) { nNow = GetTime(); LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->GetHeight(), Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), pindex)); } + pindex = chainActive.Next(pindex); } - - // After rescanning, persist Sapling note data that might have changed, e.g. nullifiers. - // Do not flush the wallet here for performance reasons. - CWalletDB walletdb(strWalletFile, "r+", false); - for (auto hash : myTxHashes) { - CWalletTx wtx = mapWallet[hash]; - if (!wtx.mapSaplingNoteData.empty()) { - if (!wtx.WriteToDisk(&walletdb)) { - LogPrintf("Rescanning... WriteToDisk failed to update Sapling note data for: %s\n", hash.ToString()); - } - } - } + + //Update all witness caches + BuildWitnessCache(chainActive.Tip(), false); ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index eb350fb40..0946d452b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -23,6 +23,7 @@ #define BITCOIN_WALLET_WALLET_H #include "amount.h" +#include "asyncrpcoperation.h" #include "coins.h" #include "key.h" #include "keystore.h" @@ -60,6 +61,11 @@ extern unsigned int nTxConfirmTarget; extern bool bSpendZeroConfChange; extern bool fSendFreeTransactions; extern bool fPayAtLeastCustomFee; +extern bool fTxDeleteEnabled; +extern bool fTxConflictDeleteEnabled; +extern int fDeleteInterval; +extern unsigned int fDeleteTransactionsAfterNBlocks; +extern unsigned int fKeepLastNTransactions; //! -paytxfee default @@ -82,6 +88,18 @@ extern unsigned int WITNESS_CACHE_SIZE; //! Size of HD seed in bytes static const size_t HD_WALLET_SEED_LENGTH = 32; +//Default Transaction Rentention N-BLOCKS +static const int DEFAULT_TX_DELETE_INTERVAL = 1000; + +//Default Transaction Rentention N-BLOCKS +static const unsigned int DEFAULT_TX_RETENTION_BLOCKS = 10000; + +//Default Retenion Last N-Transactions +static const unsigned int DEFAULT_TX_RETENTION_LASTTX = 200; + +//Amount of transactions to delete per run while syncing +static const int MAX_DELETE_TX_SIZE = 50000; + class CBlockIndex; class CCoinControl; class COutput; @@ -250,15 +268,18 @@ public: * -1 as a placeholder. The next time CWallet::ChainTip is called, we can * determine what height the witness cache for this note is valid for (even * if no witnesses were cached), and so can set the correct value in - * CWallet::IncrementNoteWitnesses and CWallet::DecrementNoteWitnesses. + * CWallet::BuildWitnessCache and CWallet::DecrementNoteWitnesses. */ int witnessHeight; - SproutNoteData() : address(), nullifier(), witnessHeight {-1} { } + //In Memory Only + bool witnessRootValidated; + + SproutNoteData() : address(), nullifier(), witnessHeight {-1}, witnessRootValidated {false} { } SproutNoteData(libzcash::SproutPaymentAddress a) : - address {a}, nullifier(), witnessHeight {-1} { } + address {a}, nullifier(), witnessHeight {-1}, witnessRootValidated {false} { } SproutNoteData(libzcash::SproutPaymentAddress a, uint256 n) : - address {a}, nullifier {n}, witnessHeight {-1} { } + address {a}, nullifier {n}, witnessHeight {-1}, witnessRootValidated {false} { } ADD_SERIALIZE_METHODS; @@ -300,6 +321,9 @@ public: libzcash::SaplingIncomingViewingKey ivk; boost::optional nullifier; + //In Memory Only + bool witnessRootValidated; + ADD_SERIALIZE_METHODS; template @@ -779,6 +803,9 @@ private: typedef TxSpendMap TxNullifiers; TxNullifiers mapTxSproutNullifiers; TxNullifiers mapTxSaplingNullifiers; + + std::vector pendingSaplingConsolidationTxs; + AsyncRPCOperationId saplingConsolidationOperationId; void AddToTransparentSpends(const COutPoint& outpoint, const uint256& wtxid); void AddToSproutSpends(const uint256& nullifier, const uint256& wtxid); @@ -793,6 +820,7 @@ public: */ int64_t nWitnessCacheSize; bool needsRescan = false; + bool fSaplingConsolidationEnabled = false; void ClearNoteWitnessCache(); @@ -800,13 +828,16 @@ public: std::set GetNullifiers(); protected: + + int SproutWitnessMinimumHeight(const uint256& nullifier, int nWitnessHeight, int nMinimumHeight); + int SaplingWitnessMinimumHeight(const uint256& nullifier, int nWitnessHeight, int nMinimumHeight); + /** * pindex is the new tip being connected. */ - void IncrementNoteWitnesses(const CBlockIndex* pindex, - const CBlock* pblock, - SproutMerkleTree& sproutTree, - SaplingMerkleTree& saplingTree); + int VerifyAndSetInitialWitness(const CBlockIndex* pindex, bool witnessOnly); + void BuildWitnessCache(const CBlockIndex* pindex, bool witnessOnly); + /** * pindex is the old tip being disconnected. */ @@ -1001,8 +1032,11 @@ public: bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, std::vector vCoins, std::set >& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; + unsigned int GetSpendDepth(const uint256& hash, unsigned int n) const; bool IsSproutSpent(const uint256& nullifier) const; + unsigned int GetSproutSpendDepth(const uint256& nullifier) const; bool IsSaplingSpent(const uint256& nullifier) const; + unsigned int GetSaplingSpendDepth(const uint256& nullifier) const; bool IsLockedCoin(uint256 hash, unsigned int n) const; void LockCoin(COutPoint& output); @@ -1137,8 +1171,9 @@ public: void MarkDirty(); bool UpdateNullifierNoteMap(); void UpdateNullifierNoteMapWithTx(const CWalletTx& wtx); + void UpdateSproutNullifierNoteMapWithTx(CWalletTx& wtx); void UpdateSaplingNullifierNoteMapWithTx(CWalletTx& wtx); - void UpdateSaplingNullifierNoteMapForBlock(const CBlock* pblock); + void UpdateNullifierNoteMapForBlock(const CBlock* pblock); bool AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletDB* pwalletdb); void EraseFromWallet(const uint256 &hash); void SyncTransaction(const CTransaction& tx, const CBlock* pblock); @@ -1148,6 +1183,10 @@ public: std::vector commitments, std::vector>& witnesses, uint256 &final_anchor); + void ReorderWalletTransactions(std::map, CWalletTx> &mapSorted, int64_t &maxOrderPos); + void UpdateWalletTransactionOrder(std::map, CWalletTx> &mapSorted, bool resetOrder); + void DeleteTransactions(std::vector &removeTxs); + void DeleteWalletTransactions(const CBlockIndex* pindex); int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime); @@ -1215,6 +1254,8 @@ public: CAmount GetCredit(const CTransaction& tx, const isminefilter& filter) const; CAmount GetChange(const CTransaction& tx) const; void ChainTip(const CBlockIndex *pindex, const CBlock *pblock, SproutMerkleTree sproutTree, SaplingMerkleTree saplingTree, bool added); + void RunSaplingConsolidation(int blockHeight); + void CommitConsolidationTx(const CTransaction& tx); /** Saves witness caches and best block locator to disk. */ void SetBestChain(const CBlockLocator& loc); std::set> GetNullifiersForAddresses(const std::set & addresses); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index c29427b74..952a76735 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1208,6 +1208,12 @@ bool BackupWallet(const CWallet& wallet, const string& strDest) return false; } +bool CWalletDB::Compact(CDBEnv& dbenv, const std::string& strFile) +{ + bool fSuccess = dbenv.Compact(strFile); + return fSuccess; +} + // // Try to (very carefully!) recover wallet.dat if there is a problem. // diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 2c57ec1e8..9069e3a08 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -189,6 +189,7 @@ public: DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(CWallet* pwallet, std::vector& vTxHash, std::vector& vWtx); DBErrors ZapWalletTx(CWallet* pwallet, std::vector& vWtx); + static bool Compact(CDBEnv& dbenv, const std::string& strFile); static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); static bool Recover(CDBEnv& dbenv, const std::string& filename);