diff --git a/src/chain.cpp b/src/chain.cpp index 1e87b22e1..dfa993f38 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -19,12 +19,57 @@ ******************************************************************************/ #include "chain.h" +#include "main.h" +#include "txdb.h" using namespace std; /** * CChain implementation */ +void CBlockIndex::TrimSolution() +{ + AssertLockHeld(cs_main); + + // We can correctly trim a solution as soon as the block index entry has been added + // to leveldb. Updates to the block index entry (to update validity status) will be + // handled by re-reading the solution from the existing db entry. It does not help to + // try to avoid these reads by gating trimming on the validity status: the re-reads are + // efficient anyway because of caching in leveldb, and most of them are unavoidable. + if (HasSolution()) { + std::vector empty; + nSolution.swap(empty); + } +} + +CBlockHeader CBlockIndex::GetBlockHeader() const +{ + AssertLockHeld(cs_main); + + CBlockHeader header; + header.nVersion = nVersion; + if (pprev) { + header.hashPrevBlock = pprev->GetBlockHash(); + } + header.hashMerkleRoot = hashMerkleRoot; + // Hush does not have this, maybe some day + // header.hashBlockCommitments = hashBlockCommitments; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + if (HasSolution()) { + header.nSolution = nSolution; + } else { + CDiskBlockIndex dbindex; + if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw std::runtime_error("Failed to read index entry"); + } + header.nSolution = dbindex.GetSolution(); + } + return header; +} + void CChain::SetTip(CBlockIndex *pindex) { lastTip = pindex; if (pindex == NULL) { diff --git a/src/chain.h b/src/chain.h index eb94559bd..03d3b3be2 100644 --- a/src/chain.h +++ b/src/chain.h @@ -27,6 +27,7 @@ class CChainPower; #include "pow.h" #include "tinyformat.h" #include "uint256.h" +#include "util/strencodings.h" #include #include @@ -384,8 +385,14 @@ public: unsigned int nTime; unsigned int nBits; uint256 nNonce; +protected: + // The Equihash solution, if it is stored. Once we know that the block index + // entry is present in leveldb, this field can be cleared via the TrimSolution + // method to save memory. std::vector nSolution; +public: + //! (memory only) Sequential id assigned to distinguish order in which blocks are received. uint32_t nSequenceId; @@ -497,23 +504,15 @@ public: return ret; } - CBlockHeader GetBlockHeader() const - { - CBlockHeader block; - block.nVersion = nVersion; - if (pprev) - block.hashPrevBlock = pprev->GetBlockHash(); - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block; - } + //! Get the block header for this block index. Requires cs_main. + CBlockHeader GetBlockHeader() const; + + //! Clear the Equihash solution to save memory. Requires cs_main. + void TrimSolution(); uint256 GetBlockHash() const { + assert(phashBlock); return *phashBlock; } @@ -540,10 +539,11 @@ public: std::string ToString() const { - return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)", + return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)", pprev, this->chainPower.nHeight, hashMerkleRoot.ToString(), - GetBlockHash().ToString()); + phashBlock ? GetBlockHash().ToString() : "(nil)", + HasSolution()); } //! Check whether this block index entry is valid up to the passed validity level. @@ -555,6 +555,12 @@ public: return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! Is the Equihash solution stored? + bool HasSolution() const + { + return !nSolution.empty(); + } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) @@ -588,8 +594,11 @@ public: hashPrev = uint256(); } - explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) { + explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function()> getSolution) : CBlockIndex(*pindex) { hashPrev = (pprev ? pprev->GetBlockHash() : uint256()); + if (!HasSolution()) { + nSolution = getSolution(); + } } ADD_SERIALIZE_METHODS; @@ -668,21 +677,28 @@ public: } } - uint256 GetBlockHash() const + //! Get the block header for this block index. + CBlockHeader GetBlockHeader() const { - CBlockHeader block; - block.nVersion = nVersion; - block.hashPrevBlock = hashPrev; - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block.GetHash(); + CBlockHeader header; + header.nVersion = nVersion; + header.hashPrevBlock = hashPrev; + header.hashMerkleRoot = hashMerkleRoot; + // Hush does not have this, maybe some day + // header.hashBlockCommitments = hashBlockCommitments; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + header.nSolution = nSolution; + return header; + } + + std::vector GetSolution() const + { + assert(HasSolution()); + return nSolution; } - std::string ToString() const { std::string str = "CDiskBlockIndex("; @@ -692,6 +708,13 @@ public: hashPrev.ToString()); return str; } + +private: + //! This method should not be called on a CDiskBlockIndex. + void TrimSolution() + { + assert(!"called CDiskBlockIndex::TrimSolution"); + } }; /** An in-memory indexed chain of blocks. */ diff --git a/src/hush_nSPV_fullnode.h b/src/hush_nSPV_fullnode.h index 48ae589d8..5124bed35 100644 --- a/src/hush_nSPV_fullnode.h +++ b/src/hush_nSPV_fullnode.h @@ -124,7 +124,19 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height) hdr->nTime = pindex->nTime; hdr->nBits = pindex->nBits; hdr->nNonce = pindex->nNonce; - memcpy(hdr->nSolution,&pindex->nSolution[0],sizeof(hdr->nSolution)); + auto header = pindex->GetBlockHeader(); + + // if (pindex->HasSolution()) { + // header.nSolution = nSolution; + // } else { + CDiskBlockIndex dbindex; + if (!pblocktree->ReadDiskBlockIndex(pindex->GetBlockHash(), dbindex)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw std::runtime_error("Failed to read index entry"); + } + header.nSolution = dbindex.GetSolution(); + // } + memcpy(hdr->nSolution,&header.nSolution[0],sizeof(hdr->nSolution)); return(sizeof(*hdr)); } return(-1); diff --git a/src/main.cpp b/src/main.cpp index 8226e1d9c..f6d6ca270 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3689,7 +3689,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it])); setDirtyFileInfo.erase(it++); } - std::vector vBlocks; + std::vector vBlocks; vBlocks.reserve(setDirtyBlockIndex.size()); for (set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { vBlocks.push_back(*it); @@ -3698,6 +3698,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode(state, "Files to write to block index database"); } + // Now that we have written the block indices to the database, we do not + // need to store solutions for these CBlockIndex objects in memory. + // cs_main must be held here. + for (CBlockIndex *pblockindex : vBlocks) { + pblockindex->TrimSolution(); + } } // Finally remove any pruned files if (fFlushForPrune) @@ -6579,7 +6585,11 @@ void static CheckBlockIndex() } } } - // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // try { + // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // } catch (const runtime_error&) { + // assert(!"Failed to read index entry"); + // } // End: actual consistency checks. // Try descending into the first subnode. diff --git a/src/main.h b/src/main.h index 66357b3d0..cfd9fed19 100644 --- a/src/main.h +++ b/src/main.h @@ -39,6 +39,7 @@ #include "spentindex.h" #include "sync.h" #include "tinyformat.h" +#include "txdb.h" #include "txmempool.h" #include "uint256.h" diff --git a/src/rest.cpp b/src/rest.cpp index 36b4279b1..3536d790a 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -151,6 +151,7 @@ static bool rest_headers(HTTPRequest* req, std::vector headers; headers.reserve(count); + CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); { LOCK(cs_main); BlockMap::const_iterator it = mapBlockIndex.find(hash); @@ -161,11 +162,16 @@ static bool rest_headers(HTTPRequest* req, break; pindex = chainActive.Next(pindex); } - } - CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); - BOOST_FOREACH(const CBlockIndex *pindex, headers) { - ssHeader << pindex->GetBlockHeader(); + if (rf == RF_BINARY || rf == RF_HEX) { + try { + for (const CBlockIndex *pindex : headers) { + ssHeader << pindex->GetBlockHeader(); + } + } catch (const std::runtime_error&) { + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry"); + } + } } switch (rf) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1c34cd1b0..760310e60 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -147,7 +147,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) result.push_back(Pair("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex())); result.push_back(Pair("time", (int64_t)blockindex->nTime)); result.push_back(Pair("nonce", blockindex->nNonce.GetHex())); - result.push_back(Pair("solution", HexStr(blockindex->nSolution))); + result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution)); result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits))); result.push_back(Pair("difficulty", GetDifficulty(blockindex))); result.push_back(Pair("chainwork", blockindex->chainPower.chainWork.GetHex())); @@ -694,15 +694,18 @@ UniValue getblockheader(const UniValue& params, bool fHelp, const CPubKey& mypk) CBlockIndex* pblockindex = mapBlockIndex[hash]; - if (!fVerbose) - { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); - ssBlock << pblockindex->GetBlockHeader(); - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); - return strHex; + try { + if (!fVerbose) { + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + ssBlock << pblockindex->GetBlockHeader(); + std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + return strHex; + } else { + return blockheaderToJSON(pblockindex); + } + } catch (const runtime_error&) { + throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry"); } - - return blockheaderToJSON(pblockindex); } UniValue getblock(const UniValue& params, bool fHelp, const CPubKey& mypk) diff --git a/src/txdb.cpp b/src/txdb.cpp index dbba3654f..602987412 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -269,18 +269,33 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { return true; } -bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { +bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); if (fZdebug) fprintf(stderr, "%s: Writing block files\n", __FUNCTION__); - for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); + for (const auto& it : fileInfo) { + batch.Write(make_pair(DB_BLOCK_FILES, it.first), *it.second); } batch.Write(DB_LAST_BLOCK, nLastFile); if (fZdebug) fprintf(stderr, "%s: Writing block index\n", __FUNCTION__); - for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); + for (const auto& it : blockinfo) { + std::pair key = make_pair(DB_BLOCK_INDEX, it->GetBlockHash()); + try { + CDiskBlockIndex dbindex {it, [this, &key]() { + // It can happen that the index entry is written, then the Equihash solution is cleared from memory, + // then the index entry is rewritten. In that case we must read the solution from the old entry. + CDiskBlockIndex dbindex_old; + if (!Read(key, dbindex_old)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw runtime_error("Failed to read index entry"); + } + return dbindex_old.GetSolution(); + }}; + batch.Write(key, dbindex); + } catch (const runtime_error&) { + return false; + } } return WriteBatch(batch, true); } @@ -293,6 +308,11 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector& blockin return WriteBatch(batch, true); } +bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const { + return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex); +} + + bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const { return Read(make_pair(DB_TXINDEX, txid), pos); } @@ -692,7 +712,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; - pindexNew->nSolution = diskindex.nSolution; + // the Equihash solution will be loaded lazily from the dbindex entry + // pindexNew->nSolution = diskindex.nSolution; pindexNew->nStatus = diskindex.nStatus; pindexNew->nCachedBranchId = diskindex.nCachedBranchId; pindexNew->nTx = diskindex.nTx; @@ -716,7 +737,19 @@ bool CBlockTreeDB::LoadBlockIndexGuts() //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks - auto header = pindexNew->GetBlockHeader(); + CBlockHeader header; + { + LOCK(cs_main); + try { + header = pindexNew->GetBlockHeader(); + } catch (const runtime_error&) { + return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s", + diskindex.GetBlockHash().ToString()); + } + } + if (header.GetHash() != diskindex.GetBlockHash()) + return error("LoadBlockIndex(): inconsistent header vs diskindex hash: header hash = %s, diskindex hash = %s", + header.GetHash().ToString(), diskindex.GetBlockHash().ToString()); if (header.GetHash() != pindexNew->GetBlockHash()) return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", diskindex.ToString(), pindexNew->ToString()); diff --git a/src/txdb.h b/src/txdb.h index 0b0ac376c..927929960 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -26,6 +26,7 @@ #include "coins.h" #include "dbwrapper.h" +#include "chain.h" #include #include #include @@ -91,12 +92,13 @@ private: CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); public: - bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo); + bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo); bool EraseBatchSync(const std::vector& blockinfo); bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo) const; bool ReadLastBlockFile(int &nFile) const; bool WriteReindexing(bool fReindex); bool ReadReindexing(bool &fReindex) const; + bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const; bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const; bool WriteTxIndex(const std::vector > &list); bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const;