Auto merge of #2898 - str4d:2286-nu-activation-mechanism, r=str4d

Network upgrade activation mechanism

Implements ZIP 200.

Integration with `CChainParams` inspired by https://github.com/bitcoin/bitcoin/pull/7575.

Includes block index rewinding logic cherry-picked from https://github.com/bitcoin/bitcoin/pull/8149.

Closes #2286. Part of #2905.

src/Makefile.am

@@ -138,6 +138,7 @@ BITCOIN_CORE_H = \
   compressor.h \
   consensus/consensus.h \
   consensus/params.h \
+  consensus/upgrades.h \
   consensus/validation.h \
   core_io.h \
   core_memusage.h \
@@ -354,6 +355,7 @@ libbitcoin_common_a_SOURCES = \
   chainparams.cpp \
   coins.cpp \
   compressor.cpp \
+  consensus/upgrades.cpp \
   core_read.cpp \
   core_write.cpp \
   hash.cpp \

src/Makefile.gtest.include

@@ -33,6 +33,7 @@ zcash_gtest_SOURCES += \
 	gtest/test_random.cpp \
 	gtest/test_rpc.cpp \
 	gtest/test_transaction.cpp \
+	gtest/test_upgrades.cpp \
 	gtest/test_validation.cpp \
 	gtest/test_circuit.cpp \
 	gtest/test_txid.cpp \

src/chain.h

@@ -94,8 +94,14 @@ enum BlockStatus: uint32_t {
     BLOCK_FAILED_VALID       =   32, //! stage after last reached validness failed
     BLOCK_FAILED_CHILD       =   64, //! descends from failed block
     BLOCK_FAILED_MASK        =   BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD,
+
+    BLOCK_ACTIVATES_UPGRADE  =   128, //! block activates a network upgrade
 };
 
+//! Short-hand for the highest consensus validity we implement.
+//! Blocks with this validity are assumed to satisfy all consensus rules.
+static const BlockStatus BLOCK_VALID_CONSENSUS = BLOCK_VALID_SCRIPTS;
+
 /** The block chain is a tree shaped structure starting with the
  * genesis block at the root, with each block potentially having multiple
  * candidates to be the next block. A blockindex may have multiple pprev pointing
@@ -140,6 +146,11 @@ public:
     //! Verification status of this block. See enum BlockStatus
     unsigned int nStatus;
 
+    //! Branch ID corresponding to the consensus rules used to validate this block.
+    //! Only cached if block validity is BLOCK_VALID_CONSENSUS.
+    //! Persisted at each activation height, memory-only for intervening blocks.
+    boost::optional<uint32_t> nCachedBranchId;
+
     //! The anchor for the tree state up to the start of this block
     uint256 hashAnchor;
 
@@ -180,6 +191,7 @@ public:
         nTx = 0;
         nChainTx = 0;
         nStatus = 0;
+        nCachedBranchId = boost::none;
         hashAnchor = uint256();
         hashAnchorEnd = uint256();
         nSequenceId = 0;
@@ -341,6 +353,18 @@ public:
             READWRITE(VARINT(nDataPos));
         if (nStatus & BLOCK_HAVE_UNDO)
             READWRITE(VARINT(nUndoPos));
+        if (nStatus & BLOCK_ACTIVATES_UPGRADE) {
+            if (ser_action.ForRead()) {
+                uint32_t branchId;
+                READWRITE(branchId);
+                nCachedBranchId = branchId;
+            } else {
+                // nCachedBranchId must always be set if BLOCK_ACTIVATES_UPGRADE is set.
+                assert(nCachedBranchId);
+                uint32_t branchId = *nCachedBranchId;
+                READWRITE(branchId);
+            }
+        }
         READWRITE(hashAnchor);
 
         // block header

src/chainparams.cpp

@@ -90,6 +90,13 @@ public:
         consensus.nPowMaxAdjustDown = 32; // 32% adjustment down
         consensus.nPowMaxAdjustUp = 16; // 16% adjustment up
         consensus.nPowTargetSpacing = 2.5 * 60;
+        consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
+            Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
+        consensus.vUpgrades[Consensus::UPGRADE_TESTDUMMY].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
+        consensus.vUpgrades[Consensus::UPGRADE_OVERWINTER].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
+
         /**
          * The message start string should be awesome! ⓩ❤
          */
@@ -241,6 +248,13 @@ public:
         consensus.nPowMaxAdjustDown = 32; // 32% adjustment down
         consensus.nPowMaxAdjustUp = 16; // 16% adjustment up
         consensus.nPowTargetSpacing = 2.5 * 60;
+        consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
+            Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
+        consensus.vUpgrades[Consensus::UPGRADE_TESTDUMMY].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
+        consensus.vUpgrades[Consensus::UPGRADE_OVERWINTER].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
+
         pchMessageStart[0] = 0xfa;
         pchMessageStart[1] = 0x1a;
         pchMessageStart[2] = 0xf9;
@@ -341,6 +355,12 @@ public:
         consensus.nPowMaxAdjustDown = 0; // Turn off adjustment down
         consensus.nPowMaxAdjustUp = 0; // Turn off adjustment up
         consensus.nPowTargetSpacing = 2.5 * 60;
+        consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
+            Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
+        consensus.vUpgrades[Consensus::UPGRADE_TESTDUMMY].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
+        consensus.vUpgrades[Consensus::UPGRADE_OVERWINTER].nActivationHeight =
+            Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT;
 
         pchMessageStart[0] = 0xaa;
         pchMessageStart[1] = 0xe8;
@@ -394,6 +414,12 @@ public:
         vFoundersRewardAddress = { "t2FwcEhFdNXuFMv1tcYwaBJtYVtMj8b1uTg" };
         assert(vFoundersRewardAddress.size() <= consensus.GetLastFoundersRewardBlockHeight());
     }
+
+    void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivationHeight)
+    {
+        assert(idx > Consensus::BASE_SPROUT && idx < Consensus::MAX_NETWORK_UPGRADES);
+        consensus.vUpgrades[idx].nActivationHeight = nActivationHeight;
+    }
 };
 static CRegTestParams regTestParams;
 
@@ -467,3 +493,8 @@ std::string CChainParams::GetFoundersRewardAddressAtIndex(int i) const {
     assert(i >= 0 && i < vFoundersRewardAddress.size());
     return vFoundersRewardAddress[i];
 }
+
+void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivationHeight)
+{
+    regTestParams.UpdateNetworkUpgradeParameters(idx, nActivationHeight);
+}

src/chainparams.h

@@ -134,4 +134,9 @@ void SelectParams(CBaseChainParams::Network network);
  */
 bool SelectParamsFromCommandLine();
 
+/**
+ * Allows modifying the network upgrade regtest parameters.
+ */
+void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivationHeight);
+
 #endif // BITCOIN_CHAINPARAMS_H

src/consensus/params.h

@@ -9,6 +9,49 @@
 #include "uint256.h"
 
 namespace Consensus {
+
+/**
+ * Index into Params.vUpgrades and NetworkUpgradeInfo
+ *
+ * Being array indices, these MUST be numbered consecutively.
+ *
+ * The order of these indices MUST match the order of the upgrades on-chain, as
+ * several functions depends on the enum being sorted.
+ */
+enum UpgradeIndex {
+    // Sprout must be first
+    BASE_SPROUT,
+    UPGRADE_TESTDUMMY,
+    UPGRADE_OVERWINTER,
+    // NOTE: Also add new upgrades to NetworkUpgradeInfo in upgrades.cpp
+    MAX_NETWORK_UPGRADES
+};
+
+struct NetworkUpgrade {
+    /**
+     * Height of the first block for which the new consensus rules will be active
+     */
+    int nActivationHeight;
+
+    /**
+     * Special value for nActivationHeight indicating that the upgrade is always active.
+     * This is useful for testing, as it means tests don't need to deal with the activation
+     * process (namely, faking a chain of somewhat-arbitrary length).
+     *
+     * New blockchains that want to enable upgrade rules from the beginning can also use
+     * this value. However, additional care must be taken to ensure the genesis block
+     * satisfies the enabled rules.
+     */
+    static constexpr int ALWAYS_ACTIVE = 0;
+
+    /**
+     * Special value for nActivationHeight indicating that the upgrade will never activate.
+     * This is useful when adding upgrade code that has a testnet activation height, but
+     * should remain disabled on mainnet.
+     */
+    static constexpr int NO_ACTIVATION_HEIGHT = -1;
+};
+
 /**
  * Parameters that influence chain consensus.
  */
@@ -39,6 +82,7 @@ struct Params {
     int nMajorityEnforceBlockUpgrade;
     int nMajorityRejectBlockOutdated;
     int nMajorityWindow;
+    NetworkUpgrade vUpgrades[MAX_NETWORK_UPGRADES];
     /** Proof of work parameters */
     uint256 powLimit;
     int64_t nPowAveragingWindow;

src/consensus/upgrades.cpp

@@ -0,0 +1,124 @@
+// Copyright (c) 2018 The Zcash developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "consensus/upgrades.h"
+
+/**
+ * General information about each network upgrade.
+ * Ordered by Consensus::UpgradeIndex.
+ */
+const struct NUInfo NetworkUpgradeInfo[Consensus::MAX_NETWORK_UPGRADES] = {
+    {
+        /*.nBranchId =*/ 0,
+        /*.strName =*/ "Sprout",
+        /*.strInfo =*/ "The Zcash network at launch",
+    },
+    {
+        /*.nBranchId =*/ 0x74736554,
+        /*.strName =*/ "Test dummy",
+        /*.strInfo =*/ "Test dummy info",
+    },
+    {
+        /*.nBranchId =*/ 0x5ba81b19,
+        /*.strName =*/ "Overwinter",
+        /*.strInfo =*/ "TBD",
+    }
+};
+
+UpgradeState NetworkUpgradeState(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex idx)
+{
+    assert(nHeight >= 0);
+    assert(idx >= Consensus::BASE_SPROUT && idx < Consensus::MAX_NETWORK_UPGRADES);
+    auto nActivationHeight = params.vUpgrades[idx].nActivationHeight;
+
+    if (nActivationHeight == Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT) {
+        return UPGRADE_DISABLED;
+    } else if (nHeight >= nActivationHeight) {
+        // From ZIP 200:
+        //
+        // ACTIVATION_HEIGHT
+        //     The non-zero block height at which the network upgrade rules will come
+        //     into effect, and be enforced as part of the blockchain consensus.
+        //
+        //     For removal of ambiguity, the block at height ACTIVATION_HEIGHT - 1 is
+        //     subject to the pre-upgrade consensus rules, and would be the last common
+        //     block in the event of a persistent pre-upgrade branch.
+        return UPGRADE_ACTIVE;
+    } else {
+        return UPGRADE_PENDING;
+    }
+}
+
+bool NetworkUpgradeActive(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex idx)
+{
+    return NetworkUpgradeState(nHeight, params, idx) == UPGRADE_ACTIVE;
+}
+
+int CurrentEpoch(int nHeight, const Consensus::Params& params) {
+    for (auto idxInt = Consensus::MAX_NETWORK_UPGRADES - 1; idxInt >= Consensus::BASE_SPROUT; idxInt--) {
+        if (NetworkUpgradeActive(nHeight, params, Consensus::UpgradeIndex(idxInt))) {
+            return idxInt;
+        }
+    }
+}
+
+uint32_t CurrentEpochBranchId(int nHeight, const Consensus::Params& params) {
+    return NetworkUpgradeInfo[CurrentEpoch(nHeight, params)].nBranchId;
+}
+
+bool IsActivationHeight(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex idx)
+{
+    assert(idx >= Consensus::BASE_SPROUT && idx < Consensus::MAX_NETWORK_UPGRADES);
+
+    // Don't count Sprout as an activation height
+    if (idx == Consensus::BASE_SPROUT) {
+        return false;
+    }
+
+    return nHeight >= 0 && nHeight == params.vUpgrades[idx].nActivationHeight;
+}
+
+bool IsActivationHeightForAnyUpgrade(
+    int nHeight,
+    const Consensus::Params& params)
+{
+    if (nHeight < 0) {
+        return false;
+    }
+
+    // Don't count Sprout as an activation height
+    for (int idx = Consensus::BASE_SPROUT + 1; idx < Consensus::MAX_NETWORK_UPGRADES; idx++) {
+        if (nHeight == params.vUpgrades[idx].nActivationHeight)
+            return true;
+    }
+
+    return false;
+}
+
+boost::optional<int> NextActivationHeight(
+    int nHeight,
+    const Consensus::Params& params)
+{
+    if (nHeight < 0) {
+        return boost::none;
+    }
+
+    // Don't count Sprout as an activation height
+    for (auto idx = Consensus::BASE_SPROUT + 1; idx < Consensus::MAX_NETWORK_UPGRADES; idx++) {
+        if (NetworkUpgradeState(nHeight, params, Consensus::UpgradeIndex(idx)) == UPGRADE_PENDING) {
+            return params.vUpgrades[idx].nActivationHeight;
+        }
+    }
+
+    return boost::none;
+}

src/consensus/upgrades.h

@@ -0,0 +1,87 @@
+// Copyright (c) 2018 The Zcash developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef ZCASH_CONSENSUS_UPGRADES_H
+#define ZCASH_CONSENSUS_UPGRADES_H
+
+#include "consensus/params.h"
+
+#include <boost/optional.hpp>
+
+enum UpgradeState {
+    UPGRADE_DISABLED,
+    UPGRADE_PENDING,
+    UPGRADE_ACTIVE
+};
+
+struct NUInfo {
+    /** Branch ID (a random non-zero 32-bit value) */
+    uint32_t nBranchId;
+    /** User-facing name for the upgrade */
+    std::string strName;
+    /** User-facing information string about the upgrade */
+    std::string strInfo;
+};
+
+extern const struct NUInfo NetworkUpgradeInfo[];
+
+/**
+ * Checks the state of a given network upgrade based on block height.
+ * Caller must check that the height is >= 0 (and handle unknown heights).
+ */
+UpgradeState NetworkUpgradeState(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex idx);
+
+/**
+ * Returns true if the given network upgrade is active as of the given block
+ * height. Caller must check that the height is >= 0 (and handle unknown
+ * heights).
+ */
+bool NetworkUpgradeActive(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex idx);
+
+/**
+ * Returns the index of the most recent upgrade as of the given block height
+ * (corresponding to the current "epoch"). Consensus::BASE_SPROUT is the
+ * default value if no upgrades are active. Caller must check that the height
+ * is >= 0 (and handle unknown heights).
+ */
+int CurrentEpoch(int nHeight, const Consensus::Params& params);
+
+/**
+ * Returns the branch ID of the most recent upgrade as of the given block height
+ * (corresponding to the current "epoch"), or 0 if no upgrades are active.
+ * Caller must check that the height is >= 0 (and handle unknown heights).
+ */
+uint32_t CurrentEpochBranchId(int nHeight, const Consensus::Params& params);
+
+/**
+ * Returns true if the given block height is the activation height for the given
+ * upgrade.
+ */
+bool IsActivationHeight(
+    int nHeight,
+    const Consensus::Params& params,
+    Consensus::UpgradeIndex upgrade);
+
+/**
+ * Returns true if the given block height is the activation height for any upgrade.
+ */
+bool IsActivationHeightForAnyUpgrade(
+    int nHeight,
+    const Consensus::Params& params);
+
+/**
+ * Returns the activation height for the next upgrade after the given block height,
+ * or boost::none if there are no more known upgrades.
+ */
+boost::optional<int> NextActivationHeight(
+    int nHeight,
+    const Consensus::Params& params);
+
+#endif // ZCASH_CONSENSUS_UPGRADES_H

src/gtest/test_upgrades.cpp

@@ -0,0 +1,173 @@
+#include <gtest/gtest.h>
+
+#include "chainparams.h"
+#include "consensus/upgrades.h"
+
+#include <boost/optional.hpp>
+
+class UpgradesTest : public ::testing::Test {
+protected:
+    virtual void SetUp() {
+    }
+
+    virtual void TearDown() {
+        // Revert to default
+        UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
+    }
+};
+
+TEST_F(UpgradesTest, NetworkUpgradeState) {
+    SelectParams(CBaseChainParams::REGTEST);
+    const Consensus::Params& params = Params().GetConsensus();
+
+    // Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT
+    EXPECT_EQ(
+        NetworkUpgradeState(0, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_DISABLED);
+    EXPECT_EQ(
+        NetworkUpgradeState(1000000, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_DISABLED);
+
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+
+    EXPECT_EQ(
+        NetworkUpgradeState(0, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_ACTIVE);
+    EXPECT_EQ(
+        NetworkUpgradeState(1000000, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_ACTIVE);
+
+    int nActivationHeight = 100;
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, nActivationHeight);
+
+    EXPECT_EQ(
+        NetworkUpgradeState(0, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_PENDING);
+    EXPECT_EQ(
+        NetworkUpgradeState(nActivationHeight - 1, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_PENDING);
+    EXPECT_EQ(
+        NetworkUpgradeState(nActivationHeight, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_ACTIVE);
+    EXPECT_EQ(
+        NetworkUpgradeState(1000000, params, Consensus::UPGRADE_TESTDUMMY),
+        UPGRADE_ACTIVE);
+}
+
+TEST_F(UpgradesTest, CurrentEpoch) {
+    SelectParams(CBaseChainParams::REGTEST);
+    const Consensus::Params& params = Params().GetConsensus();
+    auto nBranchId = NetworkUpgradeInfo[Consensus::UPGRADE_TESTDUMMY].nBranchId;
+
+    // Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT
+    EXPECT_EQ(CurrentEpoch(0, params), Consensus::BASE_SPROUT);
+    EXPECT_EQ(CurrentEpochBranchId(0, params), 0);
+    EXPECT_EQ(CurrentEpoch(1000000, params), Consensus::BASE_SPROUT);
+    EXPECT_EQ(CurrentEpochBranchId(1000000, params), 0);
+
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+
+    EXPECT_EQ(CurrentEpoch(0, params), Consensus::UPGRADE_TESTDUMMY);
+    EXPECT_EQ(CurrentEpochBranchId(0, params), nBranchId);
+    EXPECT_EQ(CurrentEpoch(1000000, params), Consensus::UPGRADE_TESTDUMMY);
+    EXPECT_EQ(CurrentEpochBranchId(1000000, params), nBranchId);
+
+    int nActivationHeight = 100;
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, nActivationHeight);
+
+    EXPECT_EQ(CurrentEpoch(0, params), Consensus::BASE_SPROUT);
+    EXPECT_EQ(CurrentEpochBranchId(0, params), 0);
+    EXPECT_EQ(CurrentEpoch(nActivationHeight - 1, params), Consensus::BASE_SPROUT);
+    EXPECT_EQ(CurrentEpochBranchId(nActivationHeight - 1, params), 0);
+    EXPECT_EQ(CurrentEpoch(nActivationHeight, params), Consensus::UPGRADE_TESTDUMMY);
+    EXPECT_EQ(CurrentEpochBranchId(nActivationHeight, params), nBranchId);
+    EXPECT_EQ(CurrentEpoch(1000000, params), Consensus::UPGRADE_TESTDUMMY);
+    EXPECT_EQ(CurrentEpochBranchId(1000000, params), nBranchId);
+}
+
+TEST_F(UpgradesTest, IsActivationHeight) {
+    SelectParams(CBaseChainParams::REGTEST);
+    const Consensus::Params& params = Params().GetConsensus();
+
+    // Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT
+    EXPECT_FALSE(IsActivationHeight(-1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(0, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1000000, params, Consensus::UPGRADE_TESTDUMMY));
+
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+
+    EXPECT_FALSE(IsActivationHeight(-1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_TRUE(IsActivationHeight(0, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1000000, params, Consensus::UPGRADE_TESTDUMMY));
+
+    int nActivationHeight = 100;
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, nActivationHeight);
+
+    EXPECT_FALSE(IsActivationHeight(-1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(0, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(nActivationHeight - 1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_TRUE(IsActivationHeight(nActivationHeight, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(nActivationHeight + 1, params, Consensus::UPGRADE_TESTDUMMY));
+    EXPECT_FALSE(IsActivationHeight(1000000, params, Consensus::UPGRADE_TESTDUMMY));
+}
+
+TEST_F(UpgradesTest, IsActivationHeightForAnyUpgrade) {
+    SelectParams(CBaseChainParams::REGTEST);
+    const Consensus::Params& params = Params().GetConsensus();
+
+    // Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(-1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(0, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1000000, params));
+
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(-1, params));
+    EXPECT_TRUE(IsActivationHeightForAnyUpgrade(0, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1000000, params));
+
+    int nActivationHeight = 100;
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, nActivationHeight);
+
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(-1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(0, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(nActivationHeight - 1, params));
+    EXPECT_TRUE(IsActivationHeightForAnyUpgrade(nActivationHeight, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(nActivationHeight + 1, params));
+    EXPECT_FALSE(IsActivationHeightForAnyUpgrade(1000000, params));
+}
+
+TEST_F(UpgradesTest, NextActivationHeight) {
+    SelectParams(CBaseChainParams::REGTEST);
+    const Consensus::Params& params = Params().GetConsensus();
+
+    // Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT
+    EXPECT_EQ(NextActivationHeight(-1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(0, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(1000000, params), boost::none);
+
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+
+    EXPECT_EQ(NextActivationHeight(-1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(0, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(1000000, params), boost::none);
+
+    int nActivationHeight = 100;
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_TESTDUMMY, nActivationHeight);
+
+    EXPECT_EQ(NextActivationHeight(-1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(0, params), nActivationHeight);
+    EXPECT_EQ(NextActivationHeight(1, params), nActivationHeight);
+    EXPECT_EQ(NextActivationHeight(nActivationHeight - 1, params), nActivationHeight);
+    EXPECT_EQ(NextActivationHeight(nActivationHeight, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(nActivationHeight + 1, params), boost::none);
+    EXPECT_EQ(NextActivationHeight(1000000, params), boost::none);
+}

src/init.cpp

@@ -16,6 +16,7 @@
 #endif
 #include "checkpoints.h"
 #include "compat/sanity.h"
+#include "consensus/upgrades.h"
 #include "consensus/validation.h"
 #include "httpserver.h"
 #include "httprpc.h"
@@ -44,8 +45,10 @@
 #include <signal.h>
 #endif
 
+#include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
 #include <boost/bind.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/function.hpp>
@@ -448,6 +451,7 @@ std::string HelpMessage(HelpMessageMode mode)
         strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages");
         strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1));
         strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0));
+        strUsage += HelpMessageOpt("-nuparams=hexBranchId:activationHeight", "Use given activation height for specified network upgrade (regtest-only)");
     }
     string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, http, libevent, lock, mempool, net, partitioncheck, pow, proxy, prune, "
                              "rand, reindex, rpc, selectcoins, tor, zmq, zrpc, zrpcunsafe (implies zrpc)"; // Don't translate these
@@ -1035,6 +1039,38 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
         }
     }
 
+    if (!mapMultiArgs["-nuparams"].empty()) {
+        // Allow overriding network upgrade parameters for testing
+        if (Params().NetworkIDString() != "regtest") {
+            return InitError("Network upgrade parameters may only be overridden on regtest.");
+        }
+        const vector<string>& deployments = mapMultiArgs["-nuparams"];
+        for (auto i : deployments) {
+            std::vector<std::string> vDeploymentParams;
+            boost::split(vDeploymentParams, i, boost::is_any_of(":"));
+            if (vDeploymentParams.size() != 2) {
+                return InitError("Network upgrade parameters malformed, expecting hexBranchId:activationHeight");
+            }
+            int nActivationHeight;
+            if (!ParseInt32(vDeploymentParams[1], &nActivationHeight)) {
+                return InitError(strprintf("Invalid nActivationHeight (%s)", vDeploymentParams[1]));
+            }
+            bool found = false;
+            // Exclude Sprout from upgrades
+            for (auto i = Consensus::BASE_SPROUT + 1; i < Consensus::MAX_NETWORK_UPGRADES; ++i)
+            {
+                if (vDeploymentParams[0].compare(HexInt(NetworkUpgradeInfo[i].nBranchId)) == 0) {
+                    UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex(i), nActivationHeight);
+                    found = true;
+                    LogPrintf("Setting network upgrade activation parameters for %s to height=%d\n", vDeploymentParams[0], nActivationHeight);
+                    break;
+                }
+            }
+            if (!found) {
+                return InitError(strprintf("Invalid network upgrade (%s)", vDeploymentParams[0]));
+            }
+        }
+    }
 
     // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log
 
@@ -1386,6 +1422,14 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
                     break;
                 }
 
+                if (!fReindex) {
+                    uiInterface.InitMessage(_("Rewinding blocks if needed..."));
+                    if (!RewindBlockIndex(chainparams)) {
+                        strLoadError = _("Unable to rewind the database to a pre-upgrade state. You will need to redownload the blockchain");
+                        break;
+                    }
+                }
+
                 uiInterface.InitMessage(_("Verifying blocks..."));
                 if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) {
                     LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n",

src/main.cpp

@@ -13,6 +13,7 @@
 #include "chainparams.h"
 #include "checkpoints.h"
 #include "checkqueue.h"
+#include "consensus/upgrades.h"
 #include "consensus/validation.h"
 #include "deprecation.h"
 #include "init.h"
@@ -543,6 +544,9 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc
             CBlockIndex* pindex = (*mi).second;
             if (chain.Contains(pindex))
                 return pindex;
+            if (pindex->GetAncestor(chain.Height()) == chain.Tip()) {
+                return chain.Tip();
+            }
         }
     }
     return chain.Genesis();
@@ -2219,6 +2223,17 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
             pindex->nStatus |= BLOCK_HAVE_UNDO;
         }
 
+        // Now that all consensus rules have been validated, set nCachedBranchId.
+        // Move this if BLOCK_VALID_CONSENSUS is ever altered.
+        static_assert(BLOCK_VALID_CONSENSUS == BLOCK_VALID_SCRIPTS,
+            "nCachedBranchId must be set after all consensus rules have been validated.");
+        if (IsActivationHeightForAnyUpgrade(pindex->nHeight, Params().GetConsensus())) {
+            pindex->nStatus |= BLOCK_ACTIVATES_UPGRADE;
+            pindex->nCachedBranchId = CurrentEpochBranchId(pindex->nHeight, chainparams.GetConsensus());
+        } else if (pindex->pprev) {
+            pindex->nCachedBranchId = pindex->pprev->nCachedBranchId;
+        }
+
         pindex->RaiseValidity(BLOCK_VALID_SCRIPTS);
         setDirtyBlockIndex.insert(pindex);
     }
@@ -2405,7 +2420,7 @@ void static UpdateTip(CBlockIndex *pindexNew) {
 }
 
 /** Disconnect chainActive's tip. */
-bool static DisconnectTip(CValidationState &state) {
+bool static DisconnectTip(CValidationState &state, bool fBare = false) {
     CBlockIndex *pindexDelete = chainActive.Tip();
     assert(pindexDelete);
     mempool.check(pcoinsTip);
@@ -2427,21 +2442,25 @@ bool static DisconnectTip(CValidationState &state) {
     // Write the chain state to disk, if necessary.
     if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
         return false;
-    // Resurrect mempool transactions from the disconnected block.
-    BOOST_FOREACH(const CTransaction &tx, block.vtx) {
-        // ignore validation errors in resurrected transactions
-        list<CTransaction> removed;
-        CValidationState stateDummy;
-        if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL))
-            mempool.remove(tx, removed, true);
-    }
-    if (anchorBeforeDisconnect != anchorAfterDisconnect) {
-        // The anchor may not change between block disconnects,
-        // in which case we don't want to evict from the mempool yet!
-        mempool.removeWithAnchor(anchorBeforeDisconnect);
-    }
-    mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight);
-    mempool.check(pcoinsTip);
+
+    if (!fBare) {
+        // Resurrect mempool transactions from the disconnected block.
+        BOOST_FOREACH(const CTransaction &tx, block.vtx) {
+            // ignore validation errors in resurrected transactions
+            list<CTransaction> removed;
+            CValidationState stateDummy;
+            if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL))
+                mempool.remove(tx, removed, true);
+        }
+        if (anchorBeforeDisconnect != anchorAfterDisconnect) {
+            // The anchor may not change between block disconnects,
+            // in which case we don't want to evict from the mempool yet!
+            mempool.removeWithAnchor(anchorBeforeDisconnect);
+        }
+        mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight);
+        mempool.check(pcoinsTip);
+    }
+
     // Update chainActive and related variables.
     UpdateTip(pindexDelete->pprev);
     // Get the current commitment tree
@@ -3555,6 +3574,19 @@ bool static LoadBlockIndexDB()
                 pindex->nChainSproutValue = pindex->nSproutValue;
             }
         }
+        // Construct in-memory chain of branch IDs.
+        // Relies on invariant: a block that does not activate a network upgrade
+        // will always be valid under the same consensus rules as its parent.
+        // Genesis block has a branch ID of zero by definition, but has no
+        // validity status because it is side-loaded into a fresh chain.
+        // Activation blocks will have branch IDs set (read from disk).
+        if (pindex->pprev) {
+            if (pindex->IsValid(BLOCK_VALID_CONSENSUS) && !pindex->nCachedBranchId) {
+                pindex->nCachedBranchId = pindex->pprev->nCachedBranchId;
+            }
+        } else {
+            pindex->nCachedBranchId = NetworkUpgradeInfo[Consensus::BASE_SPROUT].nBranchId;
+        }
         if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == NULL))
             setBlockIndexCandidates.insert(pindex);
         if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork))
@@ -3738,6 +3770,115 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth
     return true;
 }
 
+bool RewindBlockIndex(const CChainParams& params)
+{
+    LOCK(cs_main);
+
+    // RewindBlockIndex is called after LoadBlockIndex, so at this point every block
+    // index will have nCachedBranchId set based on the values previously persisted
+    // to disk. By definition, a set nCachedBranchId means that the block was
+    // fully-validated under the corresponding consensus rules. Thus we can quickly
+    // identify whether the current active chain matches our expected sequence of
+    // consensus rule changes, with two checks:
+    //
+    // - BLOCK_ACTIVATES_UPGRADE is set only on blocks that activate upgrades.
+    // - nCachedBranchId for each block matches what we expect.
+    auto sufficientlyValidated = [&params](const CBlockIndex* pindex) {
+        auto consensus = params.GetConsensus();
+        bool fFlagSet = pindex->nStatus & BLOCK_ACTIVATES_UPGRADE;
+        bool fFlagExpected = IsActivationHeightForAnyUpgrade(pindex->nHeight, consensus);
+        return fFlagSet == fFlagExpected &&
+            pindex->nCachedBranchId &&
+            *pindex->nCachedBranchId == CurrentEpochBranchId(pindex->nHeight, consensus);
+    };
+
+    int nHeight = 1;
+    while (nHeight <= chainActive.Height()) {
+        if (!sufficientlyValidated(chainActive[nHeight])) {
+            break;
+        }
+        nHeight++;
+    }
+
+    // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1
+    CValidationState state;
+    CBlockIndex* pindex = chainActive.Tip();
+    while (chainActive.Height() >= nHeight) {
+        if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) {
+            // If pruning, don't try rewinding past the HAVE_DATA point;
+            // since older blocks can't be served anyway, there's
+            // no need to walk further, and trying to DisconnectTip()
+            // will fail (and require a needless reindex/redownload
+            // of the blockchain).
+            break;
+        }
+        if (!DisconnectTip(state, true)) {
+            return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight);
+        }
+        // Occasionally flush state to disk.
+        if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC))
+            return false;
+    }
+
+    // Reduce validity flag and have-data flags.
+    // We do this after actual disconnecting, otherwise we'll end up writing the lack of data
+    // to disk before writing the chainstate, resulting in a failure to continue if interrupted.
+    for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) {
+        CBlockIndex* pindexIter = it->second;
+
+        // Note: If we encounter an insufficiently validated block that
+        // is on chainActive, it must be because we are a pruning node, and
+        // this block or some successor doesn't HAVE_DATA, so we were unable to
+        // rewind all the way.  Blocks remaining on chainActive at this point
+        // must not have their validity reduced.
+        if (!sufficientlyValidated(pindexIter) && !chainActive.Contains(pindexIter)) {
+            // Reduce validity
+            pindexIter->nStatus =
+                std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) |
+                (pindexIter->nStatus & ~BLOCK_VALID_MASK);
+            // Remove have-data flags
+            pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO);
+            // Remove branch ID
+            pindexIter->nStatus &= ~BLOCK_ACTIVATES_UPGRADE;
+            pindexIter->nCachedBranchId = boost::none;
+            // Remove storage location
+            pindexIter->nFile = 0;
+            pindexIter->nDataPos = 0;
+            pindexIter->nUndoPos = 0;
+            // Remove various other things
+            pindexIter->nTx = 0;
+            pindexIter->nChainTx = 0;
+            pindexIter->nSproutValue = boost::none;
+            pindexIter->nChainSproutValue = boost::none;
+            pindexIter->nSequenceId = 0;
+            // Make sure it gets written
+            setDirtyBlockIndex.insert(pindexIter);
+            // Update indices
+            setBlockIndexCandidates.erase(pindexIter);
+            auto ret = mapBlocksUnlinked.equal_range(pindexIter->pprev);
+            while (ret.first != ret.second) {
+                if (ret.first->second == pindexIter) {
+                    mapBlocksUnlinked.erase(ret.first++);
+                } else {
+                    ++ret.first;
+                }
+            }
+        } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) {
+            setBlockIndexCandidates.insert(pindexIter);
+        }
+    }
+
+    PruneBlockIndexCandidates();
+
+    CheckBlockIndex();
+
+    if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) {
+        return false;
+    }
+
+    return true;
+}
+
 void UnloadBlockIndex()
 {
     LOCK(cs_main);

src/main.h

@@ -436,6 +436,13 @@ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBloc
 
 
 
+/**
+ * When there are blocks in the active chain with missing data (e.g. if the
+ * activation height and branch ID of a particular upgrade have been altered),
+ * rewind the chainstate and remove them from the block index.
+ */
+bool RewindBlockIndex(const CChainParams& params);
+
 class CBlockFileInfo
 {
 public:

src/txdb.cpp

@@ -309,6 +309,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
                 pindexNew->nNonce         = diskindex.nNonce;
                 pindexNew->nSolution      = diskindex.nSolution;
                 pindexNew->nStatus        = diskindex.nStatus;
+                pindexNew->nCachedBranchId = diskindex.nCachedBranchId;
                 pindexNew->nTx            = diskindex.nTx;
                 pindexNew->nSproutValue   = diskindex.nSproutValue;
 

src/utilstrencodings.cpp

@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <errno.h>
+#include <iomanip>
 #include <limits>
 
 using namespace std;
@@ -46,6 +47,13 @@ string SanitizeFilename(const string& str)
     return strResult;
 }
 
+std::string HexInt(uint32_t val)
+{
+    std::stringstream ss;
+    ss << std::setfill('0') << std::setw(sizeof(uint32_t) * 2) << std::hex << val;
+    return ss.str();
+}
+
 const signed char p_util_hexdigit[256] =
 { -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
   -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,

src/utilstrencodings.h

@@ -24,6 +24,7 @@
 
 std::string SanitizeFilename(const std::string& str);
 std::string SanitizeString(const std::string& str);
+std::string HexInt(uint32_t val);
 std::vector<unsigned char> ParseHex(const char* psz);
 std::vector<unsigned char> ParseHex(const std::string& str);
 signed char HexDigit(char c);