Merge pull request #63 from MyHush/zindex_anonset

Zindex anonset

src/chain.cpp

@@ -1,5 +1,7 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2019-2020 The Hush developers
+
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 

src/chain.h

@@ -276,6 +276,9 @@ public:
     //! inputs and outputs.
     int64_t nShieldedTx;
 
+    //! (memory only) Number of shielded outputs in the block up to and including this block.
+    int64_t nShieldedOutputs;
+
     //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined
     //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent
     // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc...
@@ -326,6 +329,9 @@ public:
     //! inputs and outputs.
     int64_t nChainShieldedTx;
 
+    //! (memory only) Number of shielded outputs in the chain up to and including this block.
+    int64_t nChainShieldedOutputs;
+
     //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined
     //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent
     // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc...
@@ -415,22 +421,26 @@ public:
         nTx = 0;
         nChainTx = 0;
 
+        // Shieldex Index chain stats
         nChainPayments = 0;
         nChainShieldedTx = 0;
         nChainShieldingTx = 0;
         nChainDeshieldingTx = 0;
         nChainNotarizations = 0;
         nChainFullyShieldedTx = 0;
+        nChainShieldedOutputs = 0;
         nChainShieldedPayments = 0;
         nChainShieldingPayments = 0;
         nChainDeshieldingPayments = 0;
         nChainFullyShieldedPayments = 0;
 
+        // Shieldex Index stats
         nPayments = 0;
         nShieldedTx = 0;
         nShieldingTx = 0;
         nNotarizations = 0;
         nDeshieldingTx = 0;
+        nShieldedOutputs = 0;
         nFullyShieldedTx = 0;
         nShieldedPayments = 0;
         nShieldingPayments = 0;
@@ -654,27 +664,21 @@ public:
         if ((s.GetType() & SER_DISK) && (nVersion >= SAPLING_VALUE_VERSION)) {
             READWRITE(nSaplingValue);
         }
-		/*
-        if ( (s.GetType() & SER_DISK) && (is_STAKED(ASSETCHAINS_SYMBOL) != 0) && ASSETCHAINS_NOTARY_PAY[0] != 0 )
-        {
-            READWRITE(nNotaryPay);
-            READWRITE(segid);
-        }
-		*/
 
         // These values only serialized when -zindex enabled
+        // Order is important!
         if((s.GetType() & SER_DISK) && fZindex) {
             READWRITE(nShieldedTx);
             READWRITE(nShieldingTx);
             READWRITE(nDeshieldingTx);
             READWRITE(nFullyShieldedTx);
-
             READWRITE(nPayments);
             READWRITE(nNotarizations);
             READWRITE(nShieldedPayments);
             READWRITE(nShieldingPayments);
             READWRITE(nDeshieldingPayments);
             READWRITE(nFullyShieldedPayments);
+            READWRITE(nShieldedOutputs);
         }
     }
 

src/coins.cpp

@@ -52,6 +52,11 @@ void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) con
     nBytes += nLastUsedByte;
 }
 
+CNullifiersMap CCoinsViewCache::getNullifiers()
+{
+    return cacheSaplingNullifiers;
+}
+
 bool CCoins::Spend(uint32_t nPos) 
 {
     if (nPos >= vout.size() || vout[nPos].IsNull())

src/coins.h

@@ -1,6 +1,6 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
-// Copyright (c) 2019      The Hush developers
+// Copyright (c) 2019-2020 The Hush developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -490,8 +490,9 @@ public:
     CCoinsViewCache(CCoinsView *baseIn);
     ~CCoinsViewCache();
 
+    CNullifiersMap getNullifiers();
+
     // Standard CCoinsView methods
-    //static CLaunchMap &LaunchMap() { return launchMap; }
     bool GetSproutAnchorAt(const uint256 &rt, SproutMerkleTree &tree) const;
     bool GetSaplingAnchorAt(const uint256 &rt, SaplingMerkleTree &tree) const;
     bool GetNullifier(const uint256 &nullifier, ShieldedType type) const;

src/main.cpp

@@ -4717,7 +4717,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl
     CAmount sproutValue       = 0;
     CAmount saplingValue      = 0;
     bool isShieldedTx         = false;
-    unsigned int nShieldedSpends=0,nShieldedOutputs=0,nPayments=0;
+    unsigned int nShieldedSpends=0,nShieldedOutputs=0,nPayments=0, nShieldedOutputsInBlock=0;
     unsigned int nShieldedTx=0,nFullyShieldedTx=0,nDeshieldingTx=0,nShieldingTx=0;
     unsigned int nShieldedPayments=0,nFullyShieldedPayments=0,nShieldingPayments=0,nDeshieldingPayments=0;
     unsigned int nNotarizations=0;
@@ -4805,7 +4805,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl
             // No shielded payments, add transparent payments minus a change address
             nPayments +=  tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size();
         }
+        // To calculate the anonset we must track the sum of zouts in every tx, in every block. -- Duke
+        nShieldedOutputsInBlock += nShieldedOutputs;
     }
+
     pindexNew->nSproutValue = sproutValue;
     pindexNew->nChainSproutValue = boost::none;
     pindexNew->nSaplingValue = saplingValue;
@@ -4819,6 +4822,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl
     if (fZindex) {
         pindexNew->nPayments              = nPayments;
         pindexNew->nShieldedTx            = nShieldedTx;
+        pindexNew->nShieldedOutputs       = nShieldedOutputsInBlock;
         pindexNew->nFullyShieldedTx       = nFullyShieldedTx;
         pindexNew->nDeshieldingTx         = nDeshieldingTx;
         pindexNew->nShieldingTx           = nShieldingTx;
@@ -4842,18 +4846,19 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl
             pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx;
 
             if (fZindex) {
-                //fprintf(stderr,"%s: setting chain zstats\n", __FUNCTION__);
-                pindex->nChainNotarizations         = (pindex->pprev ? pindex->pprev->nChainNotarizations : 0) + pindex->nNotarizations;
-                pindex->nChainShieldedTx            = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx;
-                pindex->nChainFullyShieldedTx       = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx;
-                pindex->nChainShieldingTx           = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx;
-                pindex->nChainDeshieldingTx         = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx;
-
-                pindex->nChainPayments              = (pindex->pprev ? pindex->pprev->nChainPayments         : 0) + pindex->nPayments;
-                pindex->nChainShieldedPayments      = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments;
+                if (fZdebug)
+                    fprintf(stderr,"%s: setting blockchain zstats with zouts=%d\n", __FUNCTION__, nShieldedOutputsInBlock );
+                pindex->nChainNotarizations         = (pindex->pprev ? pindex->pprev->nChainNotarizations         : 0) + pindex->nNotarizations;
+                pindex->nChainShieldedTx            = (pindex->pprev ? pindex->pprev->nChainShieldedTx            : 0) + pindex->nShieldedTx;
+                pindex->nChainShieldedOutputs       = (pindex->pprev ? pindex->pprev->nChainShieldedOutputs       : 0) + pindex->nShieldedOutputs;
+                pindex->nChainFullyShieldedTx       = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx       : 0) + pindex->nFullyShieldedTx;
+                pindex->nChainShieldingTx           = (pindex->pprev ? pindex->pprev->nChainShieldingTx           : 0) + pindex->nShieldingTx;
+                pindex->nChainDeshieldingTx         = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx         : 0) + pindex->nDeshieldingTx;
+                pindex->nChainPayments              = (pindex->pprev ? pindex->pprev->nChainPayments              : 0) + pindex->nPayments;
+                pindex->nChainShieldedPayments      = (pindex->pprev ? pindex->pprev->nChainShieldedPayments      : 0) + pindex->nShieldedPayments;
                 pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments;
-                pindex->nChainShieldingPayments     = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments;
-                pindex->nChainDeshieldingPayments   = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments;
+                pindex->nChainShieldingPayments     = (pindex->pprev ? pindex->pprev->nChainShieldingPayments     : 0) + pindex->nShieldingPayments;
+                pindex->nChainDeshieldingPayments   = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments   : 0) + pindex->nDeshieldingPayments;
             }
 
             if (pindex->pprev) {
@@ -4893,7 +4898,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl
     }
 
     if (fZindex)
-        fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, FullyShieldedTx=%d, ntz=%d\n", pindexNew->GetHeight(), nShieldedPayments, nShieldedTx,  nFullyShieldedTx, nNotarizations );
+        fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, ShieldedOutputs=%d, FullyShieldedTx=%d, ntz=%d\n",
+        pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nShieldedOutputs, nFullyShieldedTx, nNotarizations );
 
     return true;
 }
@@ -6132,6 +6138,7 @@ bool static LoadBlockIndexDB()
                     if (fZindex) {
                         pindex->nChainNotarizations         = pindex->pprev->nChainNotarizations + pindex->nNotarizations;
                         pindex->nChainShieldedTx            = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx;
+                        pindex->nChainShieldedOutputs       = pindex->pprev->nChainShieldedOutputs + pindex->nShieldedOutputs;
                         pindex->nChainShieldedPayments      = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments;
                         pindex->nChainShieldingTx           = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx;
 
@@ -6159,6 +6166,7 @@ bool static LoadBlockIndexDB()
                         pindex->nChainPayments              = 0;
                         pindex->nChainNotarizations         = 0;
                         pindex->nChainShieldedTx            = 0;
+                        pindex->nChainShieldedOutputs       = 0;
                         pindex->nChainFullyShieldedTx       = 0;
                         pindex->nChainShieldedPayments      = 0;
                         pindex->nChainShieldingPayments     = 0;
@@ -6179,6 +6187,7 @@ bool static LoadBlockIndexDB()
                     pindex->nChainPayments              = pindex->nPayments;
                     pindex->nChainNotarizations         = pindex->nNotarizations;
                     pindex->nChainShieldedTx            = pindex->nShieldedTx;
+                    pindex->nChainShieldedOutputs       = pindex->nShieldedOutputs;
                     pindex->nChainShieldedPayments      = pindex->nShieldedPayments;
                     pindex->nChainShieldingTx           = pindex->nShieldingTx;
                     pindex->nChainShieldingPayments     = pindex->nShieldingPayments;

src/rpc/blockchain.cpp

@@ -1898,7 +1898,10 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk
             "{\n"
             "  \"time\": xxxxx,                           (numeric) The timestamp for the final block in the window in UNIX format.\n"
             "  \"txcount\": xxxxx,                        (numeric) The total number of transactions in the chain up to that point.\n"
+            "  \"nullifiers\": xxxxx,                     (numeric) The total number of shielded nullifiers in the chain up to that point.\n"
             "  \"shielded_txcount\": xxxxx,               (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n"
+            "  \"shielded_outputs\": xxxxx,               (numeric) The total number of shielded outputs in the chain up to that point.\n"
+            "  \"shielded_pool_size\": xxxxx,             (numeric) The total number of unspent shielded outputs, i.e. the Shielded Pool or Anonymity Set (anonset).\n"
             "  \"shielding_txcount\": xxxxx,              (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n"
             "  \"deshielding_txcount\": xxxxx,            (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n"
             "  \"fully_shielded_txcount\": xxxxx,         (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) transactions in the chain up to that point.\n"
@@ -1997,9 +2000,9 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk
         ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments);
 
         int64_t nullifierCount = pwalletMain->NullifierCount();
-        //TODO: we actually need to have zindex keep track of total number of zouts, including change
-        // Currently zpayments does not include change and will underestimate actual zpool size
-        ret.pushKV("shielded_pool_size", (int64_t)pindex->nChainShieldedPayments - nullifierCount);
+        ret.pushKV("nullifiers", (int64_t)nullifierCount);
+        ret.pushKV("shielded_pool_size", (int64_t)pindex->nChainShieldedOutputs - nullifierCount);
+        ret.pushKV("shielded_outputs", (int64_t)pindex->nChainShieldedOutputs);
     }
 
     if (blockcount > 0) {

src/rpc/server.cpp

@@ -667,6 +667,7 @@ static const CRPCCommand vRPCCommands[] =
     { "wallet",             "z_listoperationids",     &z_listoperationids,     true  },
     { "wallet",             "z_getnewaddress",        &z_getnewaddress,        true  },
     { "wallet",             "z_listaddresses",        &z_listaddresses,        true  },
+    { "wallet",             "z_listnullifiers",       &z_listnullifiers,       true  },
     { "wallet",             "z_exportkey",            &z_exportkey,            true  },
     { "wallet",             "z_importkey",            &z_importkey,            true  },
     { "wallet",             "z_exportviewingkey",     &z_exportviewingkey,     true  },

src/rpc/server.h

@@ -481,6 +481,7 @@ extern UniValue z_exportviewingkey(const UniValue& params, bool fHelp, const CPu
 extern UniValue z_importviewingkey(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcdump.cpp
 extern UniValue z_getnewaddress(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcwallet.cpp
 extern UniValue z_listaddresses(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcwallet.cpp
+extern UniValue z_listnullifiers(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcwallet.cpp
 extern UniValue z_exportwallet(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcdump.cpp
 extern UniValue z_importwallet(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcdump.cpp
 extern UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp, const CPubKey& mypk); // in rpcwallet.cpp

src/txdb.cpp

@@ -734,6 +734,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
                 pindexNew->nNotaryPay             = diskindex.nNotaryPay;
                 pindexNew->nPayments              = diskindex.nPayments;
                 pindexNew->nShieldedTx            = diskindex.nShieldedTx;
+                pindexNew->nShieldedOutputs       = diskindex.nShieldedOutputs;
                 pindexNew->nShieldedPayments      = diskindex.nShieldedPayments;
                 pindexNew->nShieldingTx           = diskindex.nShieldingTx;
                 pindexNew->nShieldingPayments     = diskindex.nShieldingPayments;

src/txdb.h

@@ -1,5 +1,6 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2019-2020 The Hush developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 

src/txmempool.cpp

@@ -888,6 +888,10 @@ bool CTxMemPool::IsFullyNotified() {
     return nRecentlyAddedSequence == nNotifiedSequence;
 }
 
+std::map<uint256, const CTransaction*> CTxMemPool::getNullifiers() {
+    return mapSaplingNullifiers;
+}
+
 CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { }
 
 bool CCoinsViewMemPool::GetNullifier(const uint256 &nf, ShieldedType type) const

src/txmempool.h

@@ -201,6 +201,7 @@ public:
      * all inputs are in the mapNextTx array). If sanity-checking is turned off,
      * check does nothing.
      */
+    std::map<uint256, const CTransaction*> getNullifiers();
     void check(const CCoinsViewCache *pcoins) const;
     void setSanityCheck(double dFrequency = 1.0) { nCheckFrequency = static_cast<uint32_t>(dFrequency * 4294967295.0); }
 

src/wallet/rpcwallet.cpp

@@ -3743,6 +3743,32 @@ UniValue z_getnewaddress(const UniValue& params, bool fHelp, const CPubKey& mypk
     }
 }
 
+UniValue z_listnullifiers(const UniValue& params, bool fHelp, const CPubKey& mypk)
+{
+    if (!EnsureWalletIsAvailable(fHelp))
+        return NullUniValue;
+
+    if (fHelp || params.size() > 0)
+        throw runtime_error(
+            "z_listnullifiers\n"
+            "\nReturns the list of Sapling nullifiers.\n"
+            "\nResult:\n"
+            "[                     (json array of string)\n"
+            "  \"nullifier\"       (string) a Sapling nullifer\n"
+            "  ,...\n"
+            "]\n"
+            "\nExamples:\n"
+            + HelpExampleCli("z_listnullifiers", "")
+            + HelpExampleRpc("z_listnullifiers", "")
+        );
+
+    LOCK2(cs_main, pwalletMain->cs_wallet);
+    UniValue ret(UniValue::VARR);
+    for (auto nullifier: mempool.getNullifiers()) {
+        ret.push_back(nullifier.first.GetHex());
+    }
+    return ret;
+}
 
 UniValue z_listaddresses(const UniValue& params, bool fHelp, const CPubKey& mypk)
 {
@@ -3752,7 +3778,7 @@ UniValue z_listaddresses(const UniValue& params, bool fHelp, const CPubKey& mypk
     if (fHelp || params.size() > 1)
         throw runtime_error(
             "z_listaddresses ( includeWatchonly )\n"
-            "\nReturns the list of Sprout and Sapling shielded addresses belonging to the wallet.\n"
+            "\nReturns the list of Sapling shielded addresses belonging to the wallet.\n"
             "\nArguments:\n"
             "1. includeWatchonly (bool, optional, default=false) Also include watchonly addresses (see 'z_importviewingkey')\n"
             "\nResult:\n"
@@ -3773,15 +3799,6 @@ UniValue z_listaddresses(const UniValue& params, bool fHelp, const CPubKey& mypk
     }
 
     UniValue ret(UniValue::VARR);
-    {
-        std::set<libzcash::SproutPaymentAddress> addresses;
-        pwalletMain->GetSproutPaymentAddresses(addresses);
-        for (auto addr : addresses) {
-            if (fIncludeWatchonly || pwalletMain->HaveSproutSpendingKey(addr)) {
-                ret.push_back(EncodePaymentAddress(addr));
-            }
-        }
-    }
     {
         std::set<libzcash::SaplingPaymentAddress> addresses;
         pwalletMain->GetSaplingPaymentAddresses(addresses);

src/wallet/wallet.cpp

@@ -613,15 +613,6 @@ std::set<std::pair<libzcash::PaymentAddress, uint256>> CWallet::GetNullifiersFor
         }
     }
     for (const auto & txPair : mapWallet) {
-        // Sprout
-        for (const auto & noteDataPair : txPair.second.mapSproutNoteData) {
-            auto & noteData = noteDataPair.second;
-            auto & nullifier = noteData.nullifier;
-            auto & address = noteData.address;
-            if (nullifier && addresses.count(address)) {
-                nullifierSet.insert(std::make_pair(address, nullifier.get()));
-            }
-        }
         // Sapling
         for (const auto & noteDataPair : txPair.second.mapSaplingNoteData) {
             auto & noteData = noteDataPair.second;
@@ -960,12 +951,38 @@ void CWallet::AddToSpends(const uint256& wtxid)
     }
 }
 
+std::set<uint256> CWallet::GetNullifiers()
+{
+    std::set<uint256> nullifierSet;
+    for (const auto & txPair : mapWallet) {
+        // Sapling
+        for (const auto & noteDataPair : txPair.second.mapSaplingNoteData) {
+            auto & noteData = noteDataPair.second;
+            auto & nullifier = noteData.nullifier;
+            if (nullifier) {
+                nullifierSet.insert(nullifier.get());
+            }
+        }
+    }
+    return nullifierSet;
+}
+
 int64_t CWallet::NullifierCount()
 {
     LOCK(cs_wallet);
-    return mapTxSaplingNullifiers.size();
+    if(fZdebug) {
+        // this is our *local* nullifier count
+        fprintf(stderr,"%s:mapTxSaplingNullifers.size=%d\n",__FUNCTION__,(int)mapTxSaplingNullifiers.size() );
+        // here be dragons
+        fprintf(stderr,"%s:mempool.getNullifiers.size=%d\n",__FUNCTION__,(int)mempool.getNullifiers().size() );
+        // this is the global nullifier count
+        fprintf(stderr,"%s:cacheSaplingNullifiers.size=%d\n",__FUNCTION__,(int)pcoinsTip->getNullifiers().size() );
+    }
+    // TODO: expose local nullifier stats, for now global only
+    return pcoinsTip->getNullifiers().size();
 }
 
+
 void CWallet::ClearNoteWitnessCache()
 {
     LOCK(cs_wallet);

src/wallet/wallet.h

@@ -797,6 +797,7 @@ public:
     void ClearNoteWitnessCache();
 
     int64_t NullifierCount();
+    std::set<uint256> GetNullifiers();
 
 protected:
     /**