diff --git a/.gitignore b/.gitignore index c28c6dd..b1cc45e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ *.o dnsseed* dnsstats.log -.idea/ -cmake-build-debug/ -dnsseed \ No newline at end of file diff --git a/Makefile b/Makefile index 04d6da6..5b04ee9 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,13 @@ CXXFLAGS = -O3 -g0 -march=native LDFLAGS = $(CXXFLAGS) -COMMIT_HASH=`git rev-parse --short HEAD 2>/dev/null` -BUILD_DATE=`date +%FT%T%z` - dnsseed: dns.o bitcoin.o netbase.o protocol.o db.o main.o util.o g++ -pthread $(LDFLAGS) -o dnsseed dns.o bitcoin.o netbase.o protocol.o db.o main.o util.o -lcrypto -%.o: %.cpp bitcoin.h netbase.h protocol.h db.h serialize.h uint256.h util.h - g++ -pthread $(CXXFLAGS) -Wno-invalid-offsetof -c -o $@ $< +%.o: %.cpp bitcoin.h netbase.h protocol.h db.h serialize.h uint256.h util.h coin.h + g++ -std=c++11 -pthread $(CXXFLAGS) -Wall -Wno-unused -Wno-sign-compare -Wno-reorder -Wno-comment -c -o $@ $< dns.o: dns.c - gcc -pthread -std=c99 $(CXXFLAGS) dns.c -c -o dns.o + gcc -pthread -std=c99 $(CXXFLAGS) dns.c -Wall -c -o dns.o %.o: %.cpp - -help: - echo ${COMMIT_HASH} - echo ${BUILD_DATE} \ No newline at end of file diff --git a/README.md b/README.md index ccc55ee..306fbb9 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,23 @@ -# hush-seeder +## HUSH Seeder -Hush-seeder is a crawler for the Hush network, which exposes a list + +hush-seeder is a crawler for the HUSH network, based on [bitcoin-seeder](https://github.com/sipa/bitcoin-seeder), which exposes a list of reliable nodes via a built-in DNS server. Features: * regularly revisits known nodes to check their availability * bans nodes after enough failures, or bad behaviour +* accepts nodes down to protocol version < 170002 to request new IP addresses from, + but only reports good (170002) nodes. * keeps statistics over (exponential) windows of 2 hours, 8 hours, 1 day and 1 week, to base decisions on. * very low memory (a few tens of megabytes) and cpu requirements. -* crawlers run in parallel (by default 24 threads simultaneously). +* crawlers run in parallel (by default 96 threads simultaneously). ## REQUIREMENTS - sudo apt-get install build-essential libboost-all-dev libssl-dev + sudo apt-get install build-essential libboost-all-dev libssl-dev ## USAGE @@ -25,14 +28,14 @@ Assuming you want to run a dns seed on dnsseed.example.com, you will need an authorative NS record in example.com's domain record, pointing to for example vps.example.com: - dig -t NS dnsseed.example.com + dig -t NS dnsseed.example.com - ;; ANSWER SECTION - dnsseed.example.com. 86400 IN NS vps.example.com. +;; ANSWER SECTION +dnsseed.example.com. 86400 IN NS vps.example.com. On the system vps.example.com, you can now run dnsseed: - ./dnsseed -h dnsseed.example.com -n vps.example.com + ./dnsseed -h dnsseed.example.com -n vps.example.com If you want the DNS server to report SOA records, please provide an e-mail address (with the @ part replaced by .) using -m. @@ -42,23 +45,24 @@ e-mail address (with the @ part replaced by .) using -m. Compiling will require boost and ssl. On debian systems, these are provided by `libboost-dev` and `libssl-dev` respectively. - make + make This will produce the `dnsseed` binary. ## RUNNING AS NON-ROOT + Typically, you'll need root privileges to listen to port 53 (name service). One solution is using an iptables rule (Linux only) to redirect it to a non-privileged port: - iptables -t nat -A PREROUTING -p udp --dport 53 -j REDIRECT --to-port 5353 + iptables -t nat -A PREROUTING -p udp --dport 53 -j REDIRECT --to-port 5353 If properly configured, this will allow you to run dnsseed in userspace, using the -p 5353 option. ## LICENSE -MIT +MIT \ No newline at end of file diff --git a/bitcoin.cpp b/bitcoin.cpp index 5f95542..2059ef0 100644 --- a/bitcoin.cpp +++ b/bitcoin.cpp @@ -1,5 +1,6 @@ #include +#include "coin.h" #include "db.h" #include "netbase.h" #include "protocol.h" @@ -50,7 +51,7 @@ class CNode { if (nHeaderStart == -1) return; unsigned int nSize = vSend.size() - nMessageStart; memcpy((char*)&vSend[nHeaderStart] + offsetof(CMessageHeader, nMessageSize), &nSize, sizeof(nSize)); - if (vSend.GetVersion() >= 209) { + if (vSend.GetVersion() >= INIT_PROTO_VERSION) { uint256 hash = Hash(vSend.begin() + nMessageStart, vSend.end()); unsigned int nChecksum = 0; memcpy(&nChecksum, &hash, sizeof(nChecksum)); @@ -80,7 +81,7 @@ class CNode { CAddress me(CService("0.0.0.0")); BeginMessage("version"); int nBestHeight = GetRequireHeight(); - string ver = "/hush-seeder:0.01/"; + string ver = "/hush-seeder:0.02/"; vSend << PROTOCOL_VERSION << nLocalServices << nTime << you << me << nLocalNonce << ver << nBestHeight; EndMessage(); } @@ -109,15 +110,15 @@ class CNode { vRecv >> addrFrom >> nNonce; if (nVersion >= 106 && !vRecv.empty()) vRecv >> strSubVer; - if (nVersion >= 209 && !vRecv.empty()) + if (nVersion >= INIT_PROTO_VERSION && !vRecv.empty()) vRecv >> nStartingHeight; - if (nVersion >= 209) { + if (nVersion >= INIT_PROTO_VERSION) { BeginMessage("verack"); EndMessage(); } vSend.SetVersion(min(nVersion, PROTOCOL_VERSION)); - if (nVersion < 209) { + if (nVersion < INIT_PROTO_VERSION) { this->vRecv.SetVersion(min(nVersion, PROTOCOL_VERSION)); GotVersion(); } @@ -186,7 +187,7 @@ class CNode { vRecv.insert(vRecv.begin(), vHeaderSave.begin(), vHeaderSave.end()); break; } - if (vRecv.GetVersion() >= 209) { + if (vRecv.GetVersion() >= INIT_PROTO_VERSION) { uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); unsigned int nChecksum = 0; memcpy(&nChecksum, &hash, sizeof(nChecksum)); @@ -208,8 +209,8 @@ public: vRecv.SetType(SER_NETWORK); vRecv.SetVersion(0); if (time(NULL) > 1329696000) { - vSend.SetVersion(209); - vRecv.SetVersion(209); + vSend.SetVersion(INIT_PROTO_VERSION); + vRecv.SetVersion(INIT_PROTO_VERSION); } } bool Run() { diff --git a/coin.h b/coin.h new file mode 100644 index 0000000..5460f18 --- /dev/null +++ b/coin.h @@ -0,0 +1,29 @@ +// Copyright (c) 2018 The Bitcoin developers +// Copyright (c) 2018 The HUSH developers +// Distributed under the MIT/X11 software license, see the accompanying +// file license.txt or http://www.opensource.org/licenses/mit-license.php. + +#ifndef __INCLUDED_COIN_H__ +#define __INCLUDED_COIN_H__ + +#include + +static const std::string mainnet_seeds[] = {"explorer.myhush.org", + "dnsseed.myhush.org", + "dnsseed.bleuzero.com", + ""}; + +static const std::string testnet_seeds[] = {"stilgar.myhush.org", + ""}; + +static const int mainnet_port = 8888; +static const int testnet_port = 18888; +static unsigned char pchMessageStart[4] = { 0x24, 0xe9, 0x27, 0x64 }; +static unsigned char pchMessageStart_testnet[4] = { 0xfa, 0x1a, 0xf9, 0xbf }; + +#define REQUIRE_VERSION 170002 +static const int minimunClientVersion = 170002; +static const int PROTOCOL_VERSION = 170002; +static const int INIT_PROTO_VERSION = 209; + +#endif // __INCLUDED_COIN_H__ \ No newline at end of file diff --git a/combine.pl b/combine.pl index 26787fb..4086e38 100644 --- a/combine.pl +++ b/combine.pl @@ -1,7 +1,6 @@ #!/usr/bin/perl -w use strict; -use warnings FATAL => 'all'; sub loadFile { my ($file) = @_; @@ -59,7 +58,7 @@ for my $file (@ARGV) { } for my $addr (sort { $res->{$b} <=> $res->{$a} } (keys %{$res})) { - if ($addr =~ /\A(\d+)\.(\d+)\.(\d+)\.(\d+):8333/) { + if ($addr =~ /\A(\d+)\.(\d+)\.(\d+)\.(\d+):8888/) { my $a = $1*0x1000000 + $2*0x10000 + $3*0x100 + $4; printf "0x%08x %s %g%%\n",$a,$addr,(1-((1-$res->{$addr}) ** (1/$n)))*100; } diff --git a/db.h b/db.h index 2be5235..d957ce7 100644 --- a/db.h +++ b/db.h @@ -6,14 +6,13 @@ #include #include +#include "coin.h" #include "netbase.h" #include "protocol.h" #include "util.h" #define MIN_RETRY 1000 -#define REQUIRE_VERSION 170002 - static inline int GetRequireHeight(const bool testnet = fTestNet) { // return testnet ? 500000 : 350000; @@ -120,7 +119,7 @@ public: } int GetBanTime() const { if (IsGood()) return 0; - if (clientVersion && clientVersion < 31900) { return 604800; } + if (clientVersion && clientVersion < minimunClientVersion) { return 604800; } if (stat1M.reliability - stat1M.weight + 1.0 < 0.15 && stat1M.count > 32) { return 30*86400; } if (stat1W.reliability - stat1W.weight + 1.0 < 0.10 && stat1W.count > 16) { return 7*86400; } if (stat1D.reliability - stat1D.weight + 1.0 < 0.05 && stat1D.count > 8) { return 1*86400; } diff --git a/main.cpp b/main.cpp index 4a1ba67..9901d5c 100644 --- a/main.cpp +++ b/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "bitcoin.h" #include "db.h" @@ -34,7 +35,7 @@ public: CDnsSeedOpts() : nThreads(96), nDnsThreads(4), nPort(53), mbox(NULL), ns(NULL), host(NULL), tor(NULL), fUseTestNet(false), fWipeBan(false), fWipeIgnore(false), ipv4_proxy(NULL), ipv6_proxy(NULL) {} void ParseCommandLine(int argc, char **argv) { - static const char *help = "hush-seeder\n" + static const char *help = "HUSH-seeder\n" "Usage: %s -h -n [-m ] [-t ] [-p ]\n" "\n" "Options:\n" @@ -81,17 +82,17 @@ public: host = optarg; break; } - + case 'm': { mbox = optarg; break; } - + case 'n': { ns = optarg; break; } - + case 't': { int n = strtol(optarg, NULL, 10); if (n > 0 && n < 1000) nThreads = n; @@ -188,19 +189,25 @@ extern "C" void* ThreadCrawler(void* data) { db.ResultMany(ips); db.Add(addr); } while(1); + return nullptr; } extern "C" int GetIPList(void *thread, char *requestedHostname, addr_t *addr, int max, int ipv4, int ipv6); class CDnsThread { public: + struct FlagSpecificData { + int nIPv4, nIPv6; + std::vector cache; + time_t cacheTime; + unsigned int cacheHits; + FlagSpecificData() : nIPv4(0), nIPv6(0), cacheTime(0), cacheHits(0) {} + }; + dns_opt_t dns_opt; // must be first const int id; - std::map > cache; - int nIPv4, nIPv6; - std::map cacheTime; - unsigned int cacheHits; - uint64_t dbQueries; + std::map perflag; + std::atomic dbQueries; std::set filterWhitelist; void cacheHit(uint64_t requestedFlags, bool force = false) { @@ -210,15 +217,16 @@ public: nets[NET_IPV6] = true; } time_t now = time(NULL); - cacheHits++; - if (force || cacheHits > (cache[requestedFlags].size()*cache[requestedFlags].size()/400) || (cacheHits*cacheHits > cache[requestedFlags].size() / 20 && (now - cacheTime[requestedFlags] > 5))) { + FlagSpecificData& thisflag = perflag[requestedFlags]; + thisflag.cacheHits++; + if (force || thisflag.cacheHits * 400 > (thisflag.cache.size()*thisflag.cache.size()) || (thisflag.cacheHits*thisflag.cacheHits * 20 > thisflag.cache.size() && (now - thisflag.cacheTime > 5))) { set ips; db.GetIPs(ips, requestedFlags, 1000, nets); dbQueries++; - cache[requestedFlags].clear(); - nIPv4 = 0; - nIPv6 = 0; - cache[requestedFlags].reserve(ips.size()); + thisflag.cache.clear(); + thisflag.nIPv4 = 0; + thisflag.nIPv6 = 0; + thisflag.cache.reserve(ips.size()); for (set::iterator it = ips.begin(); it != ips.end(); it++) { struct in_addr addr; struct in6_addr addr6; @@ -226,18 +234,18 @@ public: addr_t a; a.v = 4; memcpy(&a.data.v4, &addr, 4); - cache[requestedFlags].push_back(a); - nIPv4++; + thisflag.cache.push_back(a); + thisflag.nIPv4++; } else if ((*it).GetIn6Addr(&addr6)) { addr_t a; a.v = 6; memcpy(&a.data.v6, &addr6, 16); - cache[requestedFlags].push_back(a); - nIPv6++; + thisflag.cache.push_back(a); + thisflag.nIPv6++; } } - cacheHits = 0; - cacheTime[requestedFlags] = now; + thisflag.cacheHits = 0; + thisflag.cacheTime = now; } } @@ -250,12 +258,8 @@ public: dns_opt.cb = GetIPList; dns_opt.port = opts->nPort; dns_opt.nRequests = 0; - cache.clear(); - cacheTime.clear(); - cacheHits = 0; dbQueries = 0; - nIPv4 = 0; - nIPv6 = 0; + perflag.clear(); filterWhitelist = opts->filter_whitelist; } @@ -280,8 +284,9 @@ extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int else if (strcasecmp(requestedHostname, thread->dns_opt.host)) return 0; thread->cacheHit(requestedFlags); - unsigned int size = thread->cache[requestedFlags].size(); - unsigned int maxmax = (ipv4 ? thread->nIPv4 : 0) + (ipv6 ? thread->nIPv6 : 0); + auto& thisflag = thread->perflag[requestedFlags]; + unsigned int size = thisflag.cache.size(); + unsigned int maxmax = (ipv4 ? thisflag.nIPv4 : 0) + (ipv6 ? thisflag.nIPv6 : 0); if (max > size) max = size; if (max > maxmax) @@ -290,16 +295,16 @@ extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int while (icache[requestedFlags][j].v == 4) || - (ipv6 && thread->cache[requestedFlags][j].v == 6); + bool ok = (ipv4 && thisflag.cache[j].v == 4) || + (ipv6 && thisflag.cache[j].v == 6); if (ok) break; j++; if (j==size) j=i; } while(1); - addr[i] = thread->cache[requestedFlags][j]; - thread->cache[requestedFlags][j] = thread->cache[requestedFlags][i]; - thread->cache[requestedFlags][i] = addr[i]; + addr[i] = thisflag.cache[j]; + thisflag.cache[j] = thisflag.cache[i]; + thisflag.cache[i] = addr[i]; i++; } return max; @@ -310,6 +315,7 @@ vector dnsThread; extern "C" void* ThreadDNS(void* arg) { CDnsThread *thread = (CDnsThread*)arg; thread->run(); + return nullptr; } int StatCompare(const CAddrReport& a, const CAddrReport& b) { @@ -346,7 +352,7 @@ extern "C" void* ThreadDumper(void*) { double stat[5]={0,0,0,0,0}; for (vector::const_iterator it = v.begin(); it < v.end(); it++) { CAddrReport rep = *it; - fprintf(d, "%-47s %4d %11"PRId64" %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08"PRIx64" %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str()); + fprintf(d, "%-47s %4d %11" PRId64 " %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08" PRIx64 " %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str()); stat[0] += rep.uptime[0]; stat[1] += rep.uptime[1]; stat[2] += rep.uptime[2]; @@ -359,6 +365,7 @@ extern "C" void* ThreadDumper(void*) { fclose(ff); } } while(1); + return nullptr; } extern "C" void* ThreadStats(void*) { @@ -387,15 +394,14 @@ extern "C" void* ThreadStats(void*) { printf("%s %i/%i available (%i tried in %is, %i new, %i active), %i banned; %llu DNS requests, %llu db queries", c, stats.nGood, stats.nAvail, stats.nTracked, stats.nAge, stats.nNew, stats.nAvail - stats.nTracked - stats.nNew, stats.nBanned, (unsigned long long)requests, (unsigned long long)queries); Sleep(1000); } while(1); + return nullptr; } -static const string mainnet_seeds[] = {"explorer.myhush.org", "" }; -static const string testnet_seeds[] = {"stilgar.myhush.org", ""}; static const string *seeds = mainnet_seeds; extern "C" void* ThreadSeeder(void*) { //if (!fTestNet){ - // db.Add(CService("kjy2eqzk4zwi5zd3.onion", 8333), true); + // db.Add(CService("kjy2eqzk4zwi5zd3.onion", 8888), true); //} do { for (int i=0; seeds[i] != ""; i++) { @@ -407,6 +413,7 @@ extern "C" void* ThreadSeeder(void*) { } Sleep(1800000); } while(1); + return nullptr; } int main(int argc, char **argv) { @@ -446,10 +453,7 @@ int main(int argc, char **argv) { bool fDNS = true; if (opts.fUseTestNet) { printf("Using testnet.\n"); - pchMessageStart[0] = 0xFA; - pchMessageStart[1] = 0x1A; - pchMessageStart[2] = 0x24; - pchMessageStart[3] = 0xB6; + std::copy(std::begin(pchMessageStart_testnet), std::end(pchMessageStart_testnet), std::begin(pchMessageStart)); seeds = testnet_seeds; fTestNet = true; } diff --git a/protocol.cpp b/protocol.cpp index f477667..7d214fd 100644 --- a/protocol.cpp +++ b/protocol.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2011 The Bitcoin developers +// Copyright (c) 2018 The HUSH developers // Distributed under the MIT/X11 software license, see the accompanying // file license.txt or http://www.opensource.org/licenses/mit-license.php. @@ -22,8 +23,6 @@ static const char* ppszTypeName[] = "block", }; -unsigned char pchMessageStart[4] = { 0x24, 0xE9, 0x27, 0x64 }; - CMessageHeader::CMessageHeader() { memcpy(pchMessageStart, ::pchMessageStart, sizeof(pchMessageStart)); diff --git a/protocol.h b/protocol.h index a73ca71..503f600 100644 --- a/protocol.h +++ b/protocol.h @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2011 The Bitcoin developers +// Copyright (c) 2018 The HUSH developers // Distributed under the MIT/X11 software license, see the accompanying // file license.txt or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +11,7 @@ #ifndef __INCLUDED_PROTOCOL_H__ #define __INCLUDED_PROTOCOL_H__ +#include "coin.h" #include "netbase.h" #include "serialize.h" #include @@ -18,7 +20,7 @@ extern bool fTestNet; static inline unsigned short GetDefaultPort(const bool testnet = fTestNet) { - return testnet ? 18888 : 8888; + return testnet ? testnet_port : mainnet_port; } // @@ -28,8 +30,6 @@ static inline unsigned short GetDefaultPort(const bool testnet = fTestNet) // (4) size // (4) checksum -extern unsigned char pchMessageStart[4]; - class CMessageHeader { public: @@ -44,7 +44,7 @@ class CMessageHeader READWRITE(FLATDATA(pchMessageStart)); READWRITE(FLATDATA(pchCommand)); READWRITE(nMessageSize); - if (nVersion >= 209) + if (nVersion >= INIT_PROTO_VERSION) READWRITE(nChecksum); ) diff --git a/serialize.h b/serialize.h index 3987ccb..02096b4 100644 --- a/serialize.h +++ b/serialize.h @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2011 The Bitcoin developers +// Copyright (c) 2018 The HUSH developers // Distributed under the MIT/X11 software license, see the accompanying // file license.txt or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SERIALIZE_H @@ -19,6 +20,8 @@ #include #include +#include "coin.h" + #if defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 int64; typedef unsigned __int64 uint64; @@ -60,8 +63,6 @@ class CDataStream; class CAutoFile; static const unsigned int MAX_SIZE = 0x02000000; -static const int PROTOCOL_VERSION = 170002; - // Used to bypass the rule against non-const reference to temporary // where it makes sense with wrappers such as CFlatData or CTxDB template diff --git a/test.pl b/test.pl index b909c01..1f34b9a 100644 --- a/test.pl +++ b/test.pl @@ -1,6 +1,5 @@ #!/usr/bin/perl -use warnings FATAL => 'all'; use threads; use threads::shared; use bytes; diff --git a/uint256.h b/uint256.h index b613282..66234b3 100644 --- a/uint256.h +++ b/uint256.h @@ -322,7 +322,7 @@ public: // hex string to uint static char phexdigit[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0 }; const char* pbegin = psz; - while (phexdigit[*psz] || *psz == '0') + while (phexdigit[(unsigned char)*psz] || *psz == '0') psz++; psz--; unsigned char* p1 = (unsigned char*)pn;