Auto merge of #2372 - str4d:2355-connectblock-bench, r=nathan-at-least

Benchmark for calling ConnectBlock on a block with many inputs

Requires placing `block-107134.tar.gz` (containing the block, and a fake CoinsDB containing its inputs) into the base directory of the repository.

To facilitate generation of the fake CoinsDB, an additional field `valuesZat` has been added to `getrawtransaction` containing the integer number of zatoshis instead of a decimal number of ZEC.

Closes #2355.

qa/zcash/create_benchmark_archive.py

@@ -0,0 +1,236 @@
+import binascii
+import json
+import plyvel
+import progressbar
+import os
+import struct
+import subprocess
+import sys
+
+ZCASH_CLI = './src/zcash-cli'
+USAGE = """
+Requirements:
+- faketime
+- tar
+- %s (edit ZCASH_CLI in this script to alter the path)
+- A running mainnet zcashd using the default datadir with -txindex=1
+
+Example usage:
+
+make -C src/leveldb/
+virtualenv venv
+. venv/bin/activate
+pip install --global-option=build_ext --global-option="-L$(pwd)/src/leveldb/" --global-option="-I$(pwd)/src/leveldb/include/" plyvel
+pip install progressbar2
+LD_LIBRARY_PATH=src/leveldb python qa/zcash/create_benchmark_archive.py
+""" % ZCASH_CLI
+
+def check_deps():
+    if subprocess.call(['which', 'faketime', 'tar', ZCASH_CLI], stdout=subprocess.PIPE):
+        print USAGE
+        sys.exit()
+
+def encode_varint(n):
+    v = bytearray()
+    l = 0
+    while True:
+        v.append((n & 0x7F) | (0x80 if l else 0x00))
+        if (n <= 0x7F):
+            break
+        n = (n >> 7) - 1
+        l += 1
+    return bytes(v)[::-1]
+
+def decode_varint(v):
+    n = 0
+    for ch in range(len(v)):
+        n = (n << 7) | (ord(v[ch]) & 0x7F)
+        if (ord(v[ch]) & 0x80):
+            n += 1
+        else:
+            return n
+
+def compress_amount(n):
+    if n == 0:
+        return 0
+    e = 0
+    while (((n % 10) == 0) and e < 9):
+        n /= 10
+        e += 1
+    if e < 9:
+        d = (n % 10)
+        assert(d >= 1 and d <= 9)
+        n /= 10
+        return 1 + (n*9 + d - 1)*10 + e
+    else:
+        return 1 + (n - 1)*10 + 9
+
+OP_DUP = 0x76
+OP_EQUAL = 0x87
+OP_EQUALVERIFY = 0x88
+OP_HASH160 = 0xa9
+OP_CHECKSIG = 0xac
+def to_key_id(script):
+    if len(script) == 25 and \
+            script[0] == OP_DUP and \
+            script[1] == OP_HASH160 and \
+            script[2] == 20 and \
+            script[23] == OP_EQUALVERIFY and \
+            script[24] == OP_CHECKSIG:
+        return script[3:23]
+    return bytes()
+
+def to_script_id(script):
+    if len(script) == 23 and \
+            script[0] == OP_HASH160 and \
+            script[1] == 20 and \
+            script[22] == OP_EQUAL:
+        return script[2:22]
+    return bytes()
+
+def to_pubkey(script):
+    if len(script) == 35 and \
+            script[0] == 33 and \
+            script[34] == OP_CHECKSIG and \
+            (script[1] == 0x02 or script[1] == 0x03):
+        return script[1:34]
+    if len(script) == 67 and \
+            script[0] == 65 and \
+            script[66] == OP_CHECKSIG and \
+            script[1] == 0x04:
+        return script[1:66] # assuming is fully valid
+    return bytes()
+
+def compress_script(script):
+    result = bytearray()
+
+    key_id = to_key_id(script)
+    if key_id:
+        result.append(0x00)
+        result.extend(key_id)
+        return bytes(result)
+
+    script_id = to_script_id(script)
+    if script_id:
+        result.append(0x01)
+        result.extend(script_id)
+        return bytes(result)
+
+    pubkey = to_pubkey(script)
+    if pubkey:
+        result.append(0x00)
+        result.extend(pubkey[1:33])
+        if pubkey[0] == 0x02 or pubkey[0] == 0x03:
+            result[0] = pubkey[0]
+            return bytes(result)
+        elif pubkey[0] == 0x04:
+            result[0] = 0x04 | (pubkey[64] & 0x01)
+            return bytes(result)
+
+    size = len(script) + 6
+    result.append(encode_varint(size))
+    result.extend(script)
+    return bytes(result)
+
+def create_benchmark_archive(blk_hash):
+    blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash]))
+    print 'Height: %d' % blk['height']
+    print 'Transactions: %d' % len(blk['tx'])
+
+    os.mkdir('benchmark')
+    with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f:
+        f.write(binascii.unhexlify(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash, 'false']).strip()))
+
+    txs = [json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1'])
+                     ) for tx in blk['tx']]
+
+    js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0])
+    if js_txs:
+        print 'Block contains %d JoinSplit-containing transactions' % js_txs
+        return
+
+    inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin'] if x.has_key('txid')]
+    print 'Total inputs: %d' % len(inputs)
+
+    unique_inputs = {}
+    for i in sorted(inputs):
+        if unique_inputs.has_key(i[0]):
+            unique_inputs[i[0]].append(i[1])
+        else:
+            unique_inputs[i[0]] = [i[1]]
+    print 'Unique input transactions: %d' % len(unique_inputs)
+
+    db_path = 'benchmark/block-%d-inputs' % blk['height']
+    db = plyvel.DB(db_path, create_if_missing=True)
+    wb = db.write_batch()
+    bar = progressbar.ProgressBar(redirect_stdout=True)
+    print 'Collecting input coins for block'
+    for tx in bar(unique_inputs.keys()):
+        rawtx = json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1']))
+
+        mask_size = 0
+        mask_code = 0
+        b = 0
+        while 2+b*8 < len(rawtx['vout']):
+            zero = True
+            i = 0
+            while i < 8 and 2+b*8+i < len(rawtx['vout']):
+                if 2+b*8+i in unique_inputs[tx]:
+                    zero = False
+                i += 1
+            if not zero:
+                mask_size = b + 1
+                mask_code += 1
+            b += 1
+
+        coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0]
+        first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx]
+        second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx]
+        code = 8*(mask_code - (0 if first or second else 1)) + \
+            (1 if coinbase else 0) + \
+            (2 if first else 0) + \
+            (4 if second else 0)
+
+        coins = bytearray()
+        # Serialized format:
+        # - VARINT(nVersion)
+        coins.extend(encode_varint(rawtx['version']))
+        # - VARINT(nCode)
+        coins.extend(encode_varint(code))
+        # - unspentness bitvector, for vout[2] and further; least significant byte first
+        for b in range(mask_size):
+            avail = 0
+            i = 0
+            while i < 8 and 2+b*8+i < len(rawtx['vout']):
+                if 2+b*8+i in unique_inputs[tx]:
+                    avail |= (1 << i)
+                i += 1
+            coins.append(avail)
+        # - the non-spent CTxOuts (via CTxOutCompressor)
+        for i in range(len(rawtx['vout'])):
+            if i in unique_inputs[tx]:
+                coins.extend(encode_varint(compress_amount(int(rawtx['vout'][i]['valueZat']))))
+                coins.extend(compress_script(
+                    binascii.unhexlify(rawtx['vout'][i]['scriptPubKey']['hex'])))
+        # - VARINT(nHeight)
+        coins.extend(encode_varint(json.loads(
+            subprocess.check_output([ZCASH_CLI, 'getblockheader', rawtx['blockhash']])
+            )['height']))
+
+        db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1])
+        db_val = bytes(coins)
+        wb.put(db_key, db_val)
+
+    wb.write()
+    db.close()
+
+    # Make reproducible archive
+    os.remove('%s/LOG' % db_path)
+    archive_name = 'block-%d.tar.gz' % blk['height']
+    subprocess.check_call(['faketime', '2017-05-17T00:00:00Z', 'tar', 'czf', archive_name, '--mtime=2017-05-17T00:00:00Z', 'benchmark'])
+    print 'Created archive %s' % archive_name
+    subprocess.call(['rm', '-r', 'benchmark'])
+
+if __name__ == '__main__':
+    check_deps()
+    create_benchmark_archive('0000000007cdb809e48e51dd0b530e8f5073e0a9e9bd7ae920fe23e874658c74')

qa/zcash/performance-measurements.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-set -e
-
 DATADIR=./benchmark-datadir
+SHA256CMD="$(command -v sha256sum || echo shasum)"
+SHA256ARGS="$(command -v sha256sum >/dev/null || echo '-a 256')"
 
 function zcash_rpc {
     ./src/zcash-cli -datadir="$DATADIR" -rpcwait -rpcuser=user -rpcpassword=password -rpcport=5983 "$@"
@@ -24,7 +24,7 @@ function zcashd_generate {
 
 function zcashd_start {
     rm -rf "$DATADIR"
-    mkdir -p "$DATADIR"
+    mkdir -p "$DATADIR/regtest"
     touch "$DATADIR/zcash.conf"
     ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
     ZCASHD_PID=$!
@@ -37,7 +37,7 @@ function zcashd_stop {
 
 function zcashd_massif_start {
     rm -rf "$DATADIR"
-    mkdir -p "$DATADIR"
+    mkdir -p "$DATADIR/regtest"
     touch "$DATADIR/zcash.conf"
     rm -f massif.out
     valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
@@ -52,7 +52,7 @@ function zcashd_massif_stop {
 
 function zcashd_valgrind_start {
     rm -rf "$DATADIR"
-    mkdir -p "$DATADIR"
+    mkdir -p "$DATADIR/regtest"
     touch "$DATADIR/zcash.conf"
     rm -f valgrind.out
     valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
@@ -65,6 +65,28 @@ function zcashd_valgrind_stop {
     cat valgrind.out
 }
 
+function extract_benchmark_data {
+    if [ -f "block-107134.tar.gz" ]; then
+        # Check the hash of the archive:
+        "$SHA256CMD" $SHA256ARGS -c <<EOF
+299a36b3445a9a0631eb9eb0b9e76c3e9e7493a98d6621ffd6dc362d3d86cbe8  block-107134.tar.gz
+EOF
+        ARCHIVE_RESULT=$?
+    else
+        echo "block-107134.tar.gz not found."
+        ARCHIVE_RESULT=1
+    fi
+    if [ $ARCHIVE_RESULT -ne 0 ]; then
+        zcashd_stop
+        echo
+        echo "Please generate it using qa/zcash/create_benchmark_archive.py"
+        echo "and place it in the base directory of the repository."
+        echo "Usage details are inside the Python script."
+        exit 1
+    fi
+    tar xzf block-107134.tar.gz -C "$DATADIR/regtest"
+}
+
 # Precomputation
 case "$1" in
     *)
@@ -107,6 +129,10 @@ case "$1" in
             incnotewitnesses)
                 zcash_rpc zcbenchmark incnotewitnesses 100 "${@:3}"
                 ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 10
+                ;;
             *)
                 zcashd_stop
                 echo "Bad arguments."
@@ -141,6 +167,10 @@ case "$1" in
             incnotewitnesses)
                 zcash_rpc zcbenchmark incnotewitnesses 1 "${@:3}"
                 ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 1
+                ;;
             *)
                 zcashd_massif_stop
                 echo "Bad arguments."
@@ -176,6 +206,10 @@ case "$1" in
             incnotewitnesses)
                 zcash_rpc zcbenchmark incnotewitnesses 1 "${@:3}"
                 ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 1
+                ;;
             *)
                 zcashd_valgrind_stop
                 echo "Bad arguments."

src/rpcrawtransaction.cpp

@@ -137,6 +137,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
         const CTxOut& txout = tx.vout[i];
         UniValue out(UniValue::VOBJ);
         out.push_back(Pair("value", ValueFromAmount(txout.nValue)));
+        out.push_back(Pair("valueZat", txout.nValue));
         out.push_back(Pair("n", (int64_t)i));
         UniValue o(UniValue::VOBJ);
         ScriptPubKeyToJSON(txout.scriptPubKey, o, true);

src/txdb.cpp

@@ -65,6 +65,9 @@ void static BatchWriteHashBestAnchor(CLevelDBBatch &batch, const uint256 &hash)
     batch.Write(DB_BEST_ANCHOR, hash);
 }
 
+CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe) {
+}
+
 CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) {
 }
 

src/txdb.h

@@ -31,6 +31,7 @@ class CCoinsViewDB : public CCoinsView
 {
 protected:
     CLevelDBWrapper db;
+    CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
 public:
     CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
 

src/wallet/rpcwallet.cpp

@@ -2589,6 +2589,11 @@ UniValue zc_benchmark(const UniValue& params, bool fHelp)
         } else if (benchmarktype == "incnotewitnesses") {
             int nTxs = params[2].get_int();
             sample_times.push_back(benchmark_increment_note_witnesses(nTxs));
+        } else if (benchmarktype == "connectblockslow") {
+            if (Params().NetworkIDString() != "regtest") {
+                throw JSONRPCError(RPC_TYPE_ERROR, "Benchmark must be run in regtest mode");
+            }
+            sample_times.push_back(benchmark_connectblock_slow());
         } else {
             throw JSONRPCError(RPC_TYPE_ERROR, "Invalid benchmarktype");
         }

src/zcbenchmarks.cpp

@@ -1,4 +1,6 @@
+#include <cstdio>
 #include <future>
+#include <map>
 #include <thread>
 #include <unistd.h>
 #include <boost/filesystem.hpp>
@@ -9,6 +11,7 @@
 #include "primitives/transaction.h"
 #include "base58.h"
 #include "crypto/equihash.h"
+#include "chain.h"
 #include "chainparams.h"
 #include "consensus/validation.h"
 #include "main.h"
@@ -17,6 +20,7 @@
 #include "script/sign.h"
 #include "sodium.h"
 #include "streams.h"
+#include "txdb.h"
 #include "utiltest.h"
 #include "wallet/wallet.h"
 
@@ -322,3 +326,82 @@ double benchmark_increment_note_witnesses(size_t nTxs)
     return timer_stop(tv_start);
 }
 
+// Fake the input of a given block
+class FakeCoinsViewDB : public CCoinsViewDB {
+    uint256 hash;
+    ZCIncrementalMerkleTree t;
+
+public:
+    FakeCoinsViewDB(std::string dbName, uint256& hash) : CCoinsViewDB(dbName, 100, false, false), hash(hash) {}
+
+    bool GetAnchorAt(const uint256 &rt, ZCIncrementalMerkleTree &tree) const {
+        if (rt == t.root()) {
+            tree = t;
+            return true;
+        }
+        return false;
+    }
+
+    bool GetNullifier(const uint256 &nf) const {
+        return false;
+    }
+
+    uint256 GetBestBlock() const {
+        return hash;
+    }
+
+    uint256 GetBestAnchor() const {
+        return t.root();
+    }
+
+    bool BatchWrite(CCoinsMap &mapCoins,
+                    const uint256 &hashBlock,
+                    const uint256 &hashAnchor,
+                    CAnchorsMap &mapAnchors,
+                    CNullifiersMap &mapNullifiers) {
+        return false;
+    }
+
+    bool GetStats(CCoinsStats &stats) const {
+        return false;
+    }
+};
+
+double benchmark_connectblock_slow()
+{
+    // Test for issue 2017-05-01.a
+    SelectParams(CBaseChainParams::MAIN);
+    CBlock block;
+    FILE* fp = fopen((GetDataDir() / "benchmark/block-107134.dat").string().c_str(), "rb");
+    if (!fp) throw new std::runtime_error("Failed to open block data file");
+    CAutoFile blkFile(fp, SER_DISK, CLIENT_VERSION);
+    blkFile >> block;
+    blkFile.fclose();
+
+    // Fake its inputs
+    auto hashPrev = uint256S("00000000159a41f468e22135942a567781c3f3dc7ad62257993eb3c69c3f95ef");
+    FakeCoinsViewDB fakeDB("benchmark/block-107134-inputs", hashPrev);
+    CCoinsViewCache view(&fakeDB);
+
+    // Fake the chain
+    CBlockIndex index(block);
+    index.nHeight = 107134;
+    CBlockIndex indexPrev;
+    indexPrev.phashBlock = &hashPrev;
+    indexPrev.nHeight = index.nHeight - 1;
+    index.pprev = &indexPrev;
+    mapBlockIndex.insert(std::make_pair(hashPrev, &indexPrev));
+
+    CValidationState state;
+    struct timeval tv_start;
+    timer_start(tv_start);
+    assert(ConnectBlock(block, state, &index, view, true));
+    auto duration = timer_stop(tv_start);
+
+    // Undo alterations to global state
+    mapBlockIndex.erase(hashPrev);
+    SelectParamsFromCommandLine();
+
+    return duration;
+}
+

src/zcbenchmarks.h

@@ -15,5 +15,6 @@ extern double benchmark_verify_equihash();
 extern double benchmark_large_tx();
 extern double benchmark_try_decrypt_notes(size_t nAddrs);
 extern double benchmark_increment_note_witnesses(size_t nTxs);
+extern double benchmark_connectblock_slow();
 
 #endif