Add rpcinfo rpc to debug deadlocks

We seem to have deadlock bugs in our RPC system, most likely inherited from ZEC or BTC.
Since some Hush RPC's take longer (such as anything with Sietch protections), the deadlocks
are more likely to occur. Eventually all RPC slots are used up and no more RPC commands
can be sent to hushd. This is why the "plz_stop" feature was implemented, but that is just
a workaround to restart the server. We must find and fix the root cause.

This rpc will allow us to see when we are getting close to our maximum work queue depth
and hopefully help us learn exactly what is happening.

src/httpserver.cpp

@@ -157,6 +157,11 @@ public:
         boost::unique_lock<boost::mutex> lock(cs);
         return queue.size();
     }
+    size_t MaxDepth()
+    {
+        boost::unique_lock<boost::mutex> lock(cs);
+        return maxDepth;
+    }
 };
 
 struct HTTPPathHandler
@@ -186,6 +191,16 @@ std::vector<HTTPPathHandler> pathHandlers;
 //! Bound listening sockets
 std::vector<evhttp_bound_socket *> boundSockets;
 
+
+int getWorkQueueDepth()
+{
+    return workQueue->Depth();
+}
+int getWorkQueueMaxDepth()
+{
+    return workQueue->MaxDepth();
+}
+
 /** Check if a network address is allowed to access the HTTP server */
 static bool ClientAllowed(const CNetAddr& netaddr)
 {

src/httpserver.h

@@ -24,6 +24,9 @@ struct event_base;
 class CService;
 class HTTPRequest;
 
+int getWorkQueueDepth();
+int getWorkQueueMaxDepth();
+
 /** Initialize HTTP server.
  * Call this before RegisterHTTPHandler or EventBase().
  */

src/rpc/misc.cpp

@@ -178,6 +178,26 @@ UniValue geterablockheights(const UniValue& params, bool fHelp, const CPubKey& m
     return(ret);
 }
 
+extern int getWorkQueueDepth();
+extern int getWorkQueueMaxDepth();
+
+UniValue rpcinfo(const UniValue& params, bool fHelp, const CPubKey& mypk)
+{
+    UniValue result(UniValue::VOBJ);
+    if (fHelp || params.size() != 0) {
+        throw runtime_error(
+            "rpcinfo\n"
+            "Returns an object containing various RPC state info.\n"
+        );
+    }
+    LOCK(cs_main);
+    int depth = getWorkQueueDepth();
+
+    result.push_back(Pair("work_queue_depth", depth));
+    result.push_back(Pair("work_queue_max_depth", getWorkQueueMaxDepth() ));
+    return result;
+}
+
 UniValue getinfo(const UniValue& params, bool fHelp, const CPubKey& mypk)
 {
     uint256 notarized_hash,notarized_desttxid; int32_t prevMoMheight,notarized_height,longestchain,hushnotarized_height,txid_height;
@@ -1556,6 +1576,7 @@ static const CRPCCommand commands[] =
     { "util",               "z_validateaddress",      &z_validateaddress,      true  }, /* uses wallet if enabled */
     { "util",               "createmultisig",         &createmultisig,         true  },
     { "util",               "verifymessage",          &verifymessage,          true  },
+    { "util",               "rpcinfo",                &rpcinfo,                true  },
 
     /* Not shown in help */
     { "hidden",             "setmocktime",            &setmocktime,            true  },