From 339641ed44b892ddcf47872ca9268aeed2456ee9 Mon Sep 17 00:00:00 2001 From: jahway603 Date: Mon, 27 Feb 2023 11:44:43 -0500 Subject: [PATCH] added new files from upstream nextgen branch --- asmap-cli.py | 65 +++ asmap-tool.py | 154 +++++++ asmap.py | 815 +++++++++++++++++++++++++++++++++++ bottleneck.py | 147 +++++++ construct/construct.py | 85 ++++ remote_dumps/prepare.sh | 4 +- remote_dumps/quagga_parse.sh | 6 +- remote_dumps/setup.sh | 4 +- 8 files changed, 1273 insertions(+), 7 deletions(-) create mode 100644 asmap-cli.py create mode 100644 asmap-tool.py create mode 100644 asmap.py create mode 100644 bottleneck.py create mode 100644 construct/construct.py diff --git a/asmap-cli.py b/asmap-cli.py new file mode 100644 index 0000000..9deb5eb --- /dev/null +++ b/asmap-cli.py @@ -0,0 +1,65 @@ +import argparse +import sys +import ipaddress + +import asmap + +print("Reading entries...") +elems = [] +with open(sys.argv[1], "r") as f: + for line in f: + idx = line.find('#') + linec = line + if idx >= 0: + linec = line[:idx] + linec = linec.strip() + s = linec.split(' ') + if len(s) == 0 or (len(s) == 1 and len(s[0]) == 0): + continue + if len(s) != 2 or not s[1].startswith("AS"): + print("Line '%s' is not valid" % line) + exit(1) + asn = int(s[1][2:]) + if asmap._CODER_ASN.can_encode(asn): + try: + net = ipaddress.ip_network(s[0]) + except ValueError: + print("Network '%s' is not valid" % net) + exit() + else: + print("Skipping unencodable AS%i" % asn) + continue + prefix = asmap.net_to_prefix(net) + elems.append((prefix, asn)) + +print("Building trie...") +elems.sort(key = lambda elem: (-len(prefix), prefix, asn)) +m = asmap.ASMap() +for prefix, asn in elems: + m.update(prefix, asn) +print("Compiling...") +bindata_filled = m.to_binary(fill=True) +bindata_unfilled = m.to_binary(fill=False) + +print("Writing...") +with open("asmap-filled.dat", "wb") as f: + f.write(bindata_filled) + +with open("asmap-unfilled.dat", "wb") as f: + f.write(bindata_unfilled) + +with open("asmap-unfilled-overlap.txt", "w") as f: + for prefix, asn in m.to_entries(fill=False, overlapping=True): + f.write("%s AS%i\n" % (asmap.prefix_to_net(prefix), asn)) + +with open("asmap-unfilled-flat.txt", "w") as f: + for prefix, asn in m.to_entries(fill=False, overlapping=False): + f.write("%s AS%i\n" % (asmap.prefix_to_net(prefix), asn)) + +with open("asmap-filled-overlap.txt", "w") as f: + for prefix, asn in m.to_entries(fill=True, overlapping=True): + f.write("%s AS%i\n" % (asmap.prefix_to_net(prefix), asn)) + +with open("asmap-filled-flat.txt", "w") as f: + for prefix, asn in m.to_entries(fill=True, overlapping=False): + f.write("%s AS%i\n" % (asmap.prefix_to_net(prefix), asn)) diff --git a/asmap-tool.py b/asmap-tool.py new file mode 100644 index 0000000..dcf107b --- /dev/null +++ b/asmap-tool.py @@ -0,0 +1,154 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import sys +import ipaddress +import math + +import asmap + +def load_file(input_file, state=None): + try: + contents = input_file.read() + except OSError as err: + sys.exit("Input file '%s' cannot be read: %s." % (input_file.name, err.strerror)) + try: + bin_asmap = asmap.ASMap.from_binary(contents) + except ValueError: + bin_asmap = None + txt_error = None + entries = None + try: + txt_contents = str(contents, encoding="utf-8") + except UnicodeError: + txt_error = "invalid UTF-8" + txt_contents = None + if txt_contents is not None: + entries = [] + for line in txt_contents.split("\n"): + idx = line.find('#') + if idx >= 0: + line = line[:idx] + line = line.lstrip(' ').rstrip(' \t\r\n') + if len(line) == 0: + continue + fields = line.split(' ') + if len(fields) != 2: + txt_error = "unparseable line '%s'" % line + entries = None + break + prefix, asn = fields + if len(asn) <= 2 or asn[:2] != "AS" or any(c < '0' or c > '9' for c in asn[2:]): + txt_error = "invalid ASN '%s'" % asn + entries = None + break + try: + net = ipaddress.ip_network(prefix) + except ValueError: + txt_error = "invalid network '%s'" % prefix + entries = None + break + entries.append((asmap.net_to_prefix(net), int(asn[2:]))) + if entries is not None and bin_asmap is not None and len(contents) > 0: + sys.exit("Input file '%s' is ambiguous." % input_file.name) + if entries is not None: + if state is None: + state = asmap.ASMap() + state.update_multi(entries) + return state + if bin_asmap is not None: + if state is None: + return bin_asmap + sys.exit("Input file '%s' is binary, and cannot be applied as a patch." % input_file.name) + sys.exit("Input file '%s' is neither a valid binary asmap file nor valid text input (%s)." % (input_file.name, txt_error)) + + +def save_binary(output_file, state, fill): + contents = state.to_binary(fill=fill) + try: + output_file.write(contents) + output_file.close() + except OSError as err: + sys.exit("Output file '%s' cannot be written to: %s." % (output_file.name, err.strerror)) + +def save_text(output_file, state, fill, overlapping): + for prefix, asn in state.to_entries(fill=fill, overlapping=overlapping): + net = asmap.prefix_to_net(prefix) + try: + print("%s AS%i" % (net, asn), file=output_file) + except OSError as err: + sys.exit("Output file '%s' cannot be written to: %s." % (output_file.name, err.strerror)) + try: + output_file.close() + except OSError as err: + sys.exit("Output file '%s' cannot be written to: %s." % (output_file.name, err.strerror)) + +def main(): + parser = argparse.ArgumentParser(description="Tool for performing various operations on texual and binary asmap files.") + subparsers = parser.add_subparsers(title="valid subcommands", dest="subcommand") + + parser_encode = subparsers.add_parser("encode", help="convert asmap data to binary format") + parser_encode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true", + help="permit reassigning undefined network ranges arbitrarily to reduce size") + parser_encode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer, + help="input asmap file (text or binary); default is stdin") + parser_encode.add_argument('outfile', nargs='?', type=argparse.FileType('wb'), default=sys.stdout.buffer, + help="output binary asmap file; default is stdout") + + parser_decode = subparsers.add_parser("decode", help="convert asmap data to text format") + parser_decode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true", + help="permit reassigning undefined network ranges arbitrarily to reduce length") + parser_decode.add_argument('-n', '--nonoverlapping', dest="overlapping", default=True, action="store_false", + help="output strictly non-overallping network ranges (increases output size)") + parser_decode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer, + help="input asmap file (text or binary); default is stdin") + parser_decode.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout, + help="output text file; default is stdout") + + parser_diff = subparsers.add_parser("diff", help="compute the difference between two asmap files") + parser_diff.add_argument('-i', '--ignore-unassigned', dest="ignore_unassigned", default=False, action="store_true", + help="ignore unassigned ranges in the first input (useful when second input is filled)") + parser_diff.add_argument('-u', '--unified', dest="unified", default=False, action="store_true", + help="output diff in 'unified' format (with +- lines)") + parser_diff.add_argument('infile1', type=argparse.FileType('rb'), + help="first file to compare (text or binary)") + parser_diff.add_argument('infile2', type=argparse.FileType('rb'), + help="second file to compare (text or binary)") + + args = parser.parse_args() + if args.subcommand is None: + parser.print_help() + elif args.subcommand == "encode": + state = load_file(args.infile) + save_binary(args.outfile, state, fill=args.fill) + elif args.subcommand == "decode": + state = load_file(args.infile) + save_text(args.outfile, state, fill=args.fill, overlapping=args.overlapping) + elif args.subcommand == "diff": + state1 = load_file(args.infile1) + state2 = load_file(args.infile2) + ipv4_changed = 0 + ipv6_changed = 0 + for prefix, old_asn, new_asn in state1.diff(state2): + if args.ignore_unassigned and old_asn == 0: + continue + net = asmap.prefix_to_net(prefix) + if isinstance(net, ipaddress.IPv4Network): + ipv4_changed += 1 << (32 - net.prefixlen) + elif isinstance(net, ipaddress.IPv6Network): + ipv6_changed += 1 << (128 - net.prefixlen) + if new_asn == 0: + print("# %s was AS%i" % (net, old_asn)) + elif old_asn == 0: + print("%s AS%i # was unassigned" % (net, new_asn)) + else: + print("%s AS%i # was AS%i" % (net, new_asn, old_asn)) + print("# %i (2^%f) IPv4 addresses changed; %i (2^%f) IPv6 addresses changed" % (ipv4_changed, math.log2(ipv4_changed), ipv6_changed, math.log2(ipv6_changed))) + else: + parser.print_help() + sys.exit("No command provided.") + +if __name__ == '__main__': + main() diff --git a/asmap.py b/asmap.py new file mode 100644 index 0000000..7a605d0 --- /dev/null +++ b/asmap.py @@ -0,0 +1,815 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +""" +This module provides the ASNEntry and ASMap classes. +""" + +import copy +import ipaddress +import random +import unittest +from enum import Enum +from functools import total_ordering +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload + +def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]: + """ + Convert an IPv4 or IPv6 network to a prefix represented as a list of bits. + + IPv4 ranges are remapped to their IPv4-mapped IPv6 range (::ffff:0:0/96). + """ + num_bits = net.prefixlen + netrange = int.from_bytes(net.network_address.packed, 'big') + + # Map an IPv4 prefix into IPv6 space. + if isinstance(net, ipaddress.IPv4Network): + num_bits += 96 + netrange += 0xffff00000000 + + # Strip unused bottom bits. + assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0 + return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)] + +def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: + """The reverse operation of net_to_prefix.""" + # Convert to number + netrange = sum(b << (127 - i) for i, b in enumerate(prefix)) + num_bits = len(prefix) + assert num_bits <= 128 + + # Return IPv4 range if in ::ffff:0:0/96 + if num_bits >= 96 and (netrange >> 32) == 0xffff: + return ipaddress.IPv4Network((netrange & 0xffffffff, num_bits - 96), True) + + # Return IPv6 range otherwise. + return ipaddress.IPv6Network((netrange, num_bits), True) + +# Shortcut for (prefix, ASN) entries. +ASNEntry = Tuple[List[bool], int] + +# Shortcut for (prefix, old ASN, new ASN) entries. +ASNDiff = Tuple[List[bool], int, int] + +class _VarLenCoder: + """ + A class representing a custom variable-length binary encoder/decoder for + integers. Each object represents a different coder, with different parameters + minval and clsbits. + + The encoding is easiest to describe using an example. Let's say minval=100 and + clsbits=[4,2,2,3]. In that case: + - x in [100..115]: encoded as [0] + [4-bit BE encoding of (x-100)]. + - x in [116..119]: encoded as [1,0] + [2-bit BE encoding of (x-116)]. + - x in [120..123]: encoded as [1,1,0] + [2-bit BE encoding of (x-120)]. + - x in [124..131]: encoded as [1,1,1] + [3-bit BE encoding of (x-124)]. + + In general, every number is encoded as: + - First, k "1"-bits, where k is the class the number falls in (there is one class + per element of clsbits). + - Then, a "0"-bit, unless k is the highest class, in which case there is nothing. + - Lastly, clsbits[k] bits encoding in big endian the position in its class that + number falls into. + - Every class k consists of 2^clsbits[k] consecutive integers. k=0 starts at minval, + other classes start one past the last element of the class before it. + """ + + def __init__(self, minval: int, clsbits: List[int]): + """Construct a new _VarLenCoder.""" + self._minval = minval + self._clsbits = clsbits + self._maxval = minval + sum(1 << b for b in clsbits) - 1 + + def can_encode(self, val: int) -> bool: + """Check whether value val is in the range this coder supports.""" + return self._minval <= val <= self._maxval + + def encode(self, val: int, ret: List[int]) -> None: + """Append encoding of val onto integer list ret.""" + + assert self._minval <= val <= self._maxval + val -= self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + # If the value will not fit in class k, subtract its range from v, + # emit a "1" bit and continue with the next class. + val -= 1 << bits + ret.append(1) + else: + if k + 1 < len(self._clsbits): + # Unless we're in the last class, emit a "0" bit. + ret.append(0) + break + # And then encode v (now the position within the class) in big endian. + ret.extend((val >> (bits - 1 - b)) & 1 for b in range(bits)) + + def encode_size(self, val: int) -> int: + """Compute how many bits are needed to encode val.""" + assert self._minval <= val <= self._maxval + val -= self._minval + ret = 0 + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + val -= 1 << bits + ret += 1 + else: + ret += k + 1 < len(self._clsbits) + break + return ret + bits + + def decode(self, stream, bitpos) -> Tuple[int,int]: + """Decode a number starting at bitpos in stream, returning value and new bitpos.""" + val = self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + bit = 0 + if k + 1 < len(self._clsbits): + bit = stream[bitpos] + bitpos += 1 + if not bit: + break + val += 1 << bits + for i in range(bits): + bit = stream[bitpos] + bitpos += 1 + val += bit << (bits - 1 - i) + return val, bitpos + +# Variable-length encoders used in the binary asmap format. +_CODER_INS = _VarLenCoder(0, [0, 0, 1]) +_CODER_ASN = _VarLenCoder(1, list(range(15, 25))) +_CODER_MATCH = _VarLenCoder(2, list(range(1, 9))) +_CODER_JUMP = _VarLenCoder(17, list(range(5, 31))) + +class _Instruction(Enum): + """One instruction in the binary asmap format.""" + # A return instruction, encoded as [0], returns a constant ASN. It is followed by + # an integer using the ASN encoding. + RETURN = 0 + # A jump instruction, encoded as [1,0] inspects the next unused bit in the input + # and either continues execution (if 0), or skips a specified number of bits (if 1). + # It is followed by an integer, and then two subprograms. The integer uses jump encoding + # and corresponds to the length of the first subprogram (so it can be skipped). + JUMP = 1 + # A match instruction, encoded as [1,1,0] inspects 1 or more of the next unused bits + # in the input with its argument. If they all match, execution continues. If they do + # not, failure is returned. If a default instruction has been executed before, instead + # of failure the default instruction's argument is returned. It is followed by an + # integer in match encoding, and a subprogram. That value is at least 2 bits and at + # most 9 bits. An n-bit value signifies matching (n-1) bits in the input with the lower + # (n-1) bits in the match value. + MATCH = 2 + # A default instruction, encoded as [1,1,1] sets the default variable to its argument, + # and continues execution. It is followed by an integer in ASN encoding, and a subprogram. + DEFAULT = 3 + # Not an actual instruction, but a way to encode the empty program that fails. In the + # encoder, it is used more generally to represent the failure case inside MATCH instructions, + # which may (if used inside the context of a DEFAULT instruction) actually correspond to + # a succesful return. In this usage, they're always converted to an actual MATCH or RETURN + # before the top level is reached (see make_default below). + END = 4 + +class _BinNode: + """A class representing a (node of) the parsed binary asmap format.""" + + @overload + def __init__(self, ins: _Instruction): ... + @overload + def __init__(self, ins: _Instruction, arg1: int): ... + @overload + def __init__(self, ins: _Instruction, arg1: "_BinNode", arg2: "_BinNode"): ... + @overload + def __init__(self, ins: _Instruction, arg1: int, arg2: "_BinNode"): ... + + def __init__(self, ins: _Instruction, arg1=None, arg2=None): + """ + Construct a new asmap node. Possibilities are: + - _BinNode(_Instruction.RETURN, asn) + - _BinNode(_Instruction.JUMP, node_0, node_1) + - _BinNode(_Instruction.MATCH, val, node) + - _BinNode(_Instruction.DEFAULT, asn, node) + - _BinNode(_Instruction.END) + """ + self.ins = ins + self.arg1 = arg1 + self.arg2 = arg2 + if ins == _Instruction.RETURN: + assert isinstance(arg1, int) + assert arg2 is None + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + elif ins == _Instruction.JUMP: + assert isinstance(arg1, _BinNode) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_JUMP.encode_size(arg1.size) + + arg1.size + arg2.size) + elif ins == _Instruction.DEFAULT: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + arg2.size + elif ins == _Instruction.MATCH: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_MATCH.encode_size(arg1) + + arg2.size) + elif ins == _Instruction.END: + assert arg1 is None + assert arg2 is None + self.size = 0 + else: + assert False + + @staticmethod + def make_end() -> "_BinNode": + """Constructor for a _BinNode with just an END instruction.""" + return _BinNode(_Instruction.END) + + @staticmethod + def make_leaf(val: int) -> "_BinNode": + """Constructor for a _BinNode of just a RETURN instruction.""" + assert val is not None and val > 0 + return _BinNode(_Instruction.RETURN, val) + + @staticmethod + def make_branch(node0: "_BinNode", node1: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode corresponding to running either the node0 or node1 subprogram, + based on the next input bit. It exploits shortcuts that are possible in the encoding, + and uses either a JUMP, MATCH, or END instruction. + """ + if node0.ins == _Instruction.END and node1.ins == _Instruction.END: + return node0 + if node0.ins == _Instruction.END: + if node1.ins == _Instruction.MATCH and node1.arg1 <= 0xFF: + return _BinNode(node1.ins, node1.arg1 + (1 << node1.arg1.bit_length()), node1.arg2) + return _BinNode(_Instruction.MATCH, 3, node1) + if node1.ins == _Instruction.END: + if node0.ins == _Instruction.MATCH and node0.arg1 <= 0xFF: + return _BinNode(node0.ins, node0.arg1 + (1 << (node0.arg1.bit_length() - 1)), + node0.arg2) + return _BinNode(_Instruction.MATCH, 2, node0) + return _BinNode(_Instruction.JUMP, node0, node1) + + @staticmethod + def make_default(val: int, sub: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode that corresponds to the specified subprogram, with the specified + default value. It exploits shortcuts that are possible in the encoding, and will use + either a DEFAULT or a RETURN instruction.""" + assert val is not None and val > 0 + if sub.ins == _Instruction.END: + return _BinNode(_Instruction.RETURN, val) + if sub.ins in (_Instruction.RETURN, _Instruction.DEFAULT): + return sub + return _BinNode(_Instruction.DEFAULT, val, sub) + +@total_ordering +class ASMap: + """ + A class whose objects represent a mapping from subnets to ASNs. + + Internally the mapping is stored as a binary trie, but can be converted + from/to a list of ASNEntry objects, and from/to the binary asmap file format. + + In the trie representation, nodes are represented as bare lists for efficiency + and ease of manipulation: + - [0] means an unassigned subnet (no ASN mapping for it is present) + - [int] means a subnet mapped entirely to the specified ASN. + - [node,node] means a subnet whose lower half and upper half have different + - mappings, represented by new trie nodes. + """ + + def update(self, prefix: List[bool], asn: int) -> None: + """Update this ASMap object to map prefix to the specified asn.""" + assert asn == 0 or _CODER_ASN.can_encode(asn) + + def recurse(node: List, offset: int) -> None: + if offset == len(prefix): + # Reached the end of prefix; overwrite this node. + node.clear() + node.append(asn) + return + if len(node) == 1: + # Need to descend into a leaf node; split it up. + oldasn = node[0] + node.clear() + node.append([oldasn]) + node.append([oldasn]) + # Descend into the node. + recurse(node[prefix[offset]], offset + 1) + # If the result is two identical leaf children, merge them. + if len(node[0]) == 1 and len(node[1]) == 1 and node[0] == node[1]: + oldasn = node[0][0] + node.clear() + node.append(oldasn) + recurse(self._trie, 0) + + def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: + """Apply multiple update operations, where longer prefixes take precedence.""" + entries.sort(key=lambda entry: len(entry[0])) + for prefix, asn in entries: + self.update(prefix, asn) + + def _set_trie(self, trie) -> None: + """Set trie directly. Internal use only.""" + def recurse(node: List) -> None: + if len(node) < 2: + return + recurse(node[0]) + recurse(node[1]) + if len(node[0]) == 2: + return + if node[0] == node[1]: + if len(node[0]) == 0: + node.clear() + else: + asn = node[0][0] + node.clear() + node.append(asn) + recurse(trie) + self._trie = trie + + def __init__(self, entries: Optional[Iterable[ASNEntry]] = None) -> None: + """Construct an ASMap object from an optional list of entries.""" + self._trie = [0] + if entries is not None: + def entry_key(entry): + """Sort function that places shorter prefixes first.""" + prefix, asn = entry + return len(prefix), prefix, asn + for prefix, asn in sorted(entries, key=entry_key): + self.update(prefix, asn) + + def lookup(self, prefix: List[bool]) -> Optional[int]: + """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate.""" + node = self._trie + for bit in prefix: + if len(node) == 1: + break + node = node[bit] + if len(node) == 1: + return node[0] + return None + + def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]: + """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects.""" + prefix : List[bool] = [] + + def recurse(node: List) -> List[ASNEntry]: + ret = [] + if len(node) == 1: + if node[0] > 0: + ret = [(list(prefix), node[0])] + elif len(node) == 2: + prefix.append(False) + ret = recurse(node[0]) + prefix[-1] = True + ret += recurse(node[1]) + prefix.pop() + if fill and len(ret) > 1: + asns = set(x[1] for x in ret) + if len(asns) == 1: + ret = [(list(prefix), list(asns)[0])] + return ret + return recurse(self._trie) + + def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]: + """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap.""" + prefix : List[bool] = [] + + def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]): + if len(node) == 1 and node[0] == 0: + return {None if fill else 0: []}, True + if len(node) == 1: + return {node[0]: [], None: [(list(prefix), node[0])]}, False + ret: Dict[Optional[int], List[ASNEntry]] = {} + prefix.append(False) + left, lhole = recurse(node[0]) + prefix[-1] = True + right, rhole = recurse(node[1]) + prefix.pop() + hole = not fill and (lhole or rhole) + def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]], + res1: Optional[List[ASNEntry]]): + if res0 is not None and res1 is not None: + if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]): + ret[ctx] = res0 + res1 + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx)) + candidate(ctx, left.get(None), right.get(ctx)) + candidate(ctx, left.get(ctx), right.get(None)) + if not hole: + for ctx in list(ret): + if ctx is not None: + candidate(None, [(list(prefix), ctx)], ret[ctx]) + if None in ret: + ret = {ctx:entries for ctx, entries in ret.items() + if ctx is None or len(entries) < len(ret[None])} + if hole: + ret = {ctx:entries for ctx, entries in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + def __str__(self) -> str: + """Convert this ASMap object to a string containing Python code constructing it.""" + return f"ASMap({self._trie})" + + def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]: + """ + Convert the mappings in this ASMap object to a list of ASNEntry objects. + + Arguments: + overlapping: Permit the subnets in the resulting ASNEntry to overlap. + Setting this can result in a shorter list. + fill: Permit the resulting ASNEntry objects to cover subnets that + are unassigned in this ASMap object. Setting this can + result in a shorter list. + """ + if overlapping: + return self._to_entries_minimal(fill) + return self._to_entries_flat(fill) + + @staticmethod + def from_random(num_leaves: int = 10, max_asn: int = 6, + unassigned_prob: float = 0.5) -> "ASMap": + """ + Construct a random ASMap object, with specified: + - Number of leaves in its trie (at least 1) + - Maximum ASN value (at least 1) + - Probability for leaf nodes to be unassigned + + The number of leaves in the resulting object may be less than what is + requested. This method is mostly intended for testing. + """ + assert num_leaves >= 1 + assert max_asn >= 1 or unassigned_prob == 1 + assert _CODER_ASN.can_encode(max_asn) + assert 0.0 <= unassigned_prob <= 1.0 + trie: List = [] + leaves = [trie] + ret = ASMap() + for i in range(1, num_leaves): + idx = random.randrange(i) + leaf = leaves[idx] + lastleaf = leaves.pop() + if idx + 1 < i: + leaves[idx] = lastleaf + leaf.append([]) + leaf.append([]) + leaves.append(leaf[0]) + leaves.append(leaf[1]) + for leaf in leaves: + if random.random() >= unassigned_prob: + leaf.append(random.randrange(1, max_asn + 1)) + else: + leaf.append(0) + #pylint: disable=protected-access + ret._set_trie(trie) + return ret + + def _to_binnode(self, fill: bool = False) -> _BinNode: + """Convert a trie to a _BinNode object.""" + def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]: + if len(node) == 1 and node[0] == 0: + return {(None if fill else 0): _BinNode.make_end()}, True + if len(node) == 1: + return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False + ret: Dict[Optional[int], _BinNode] = {} + left, lhole = recurse(node[0]) + right, rhole = recurse(node[1]) + hole = (lhole or rhole) and not fill + + def candidate(ctx: Optional[int], arg1, arg2, func: Callable): + if arg1 is not None and arg2 is not None: + cand = func(arg1, arg2) + if ctx not in ret or cand.size < ret[ctx].size: + ret[ctx] = cand + + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch) + if not hole: + for ctx in set(ret) - set([None]): + candidate(None, ctx, ret[ctx], _BinNode.make_default) + if None in ret: + ret = {ctx:enc for ctx, enc in ret.items() + if ctx is None or enc.size < ret[None].size} + if hole: + ret = {ctx:enc for ctx, enc in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + @staticmethod + def _from_binnode(binnode: _BinNode) -> "ASMap": + """Construct an ASMap object from a _BinNode. Internal use only.""" + def recurse(node: _BinNode, default: int) -> List: + if node.ins == _Instruction.RETURN: + return [node.arg1] + if node.ins == _Instruction.JUMP: + return [recurse(node.arg1, default), recurse(node.arg2, default)] + if node.ins == _Instruction.MATCH: + val = node.arg1 + sub = recurse(node.arg2, default) + while val >= 2: + bit = val & 1 + val >>= 1 + if bit: + sub = [[default], sub] + else: + sub = [sub, [default]] + return sub + assert node.ins == _Instruction.DEFAULT + return recurse(node.arg2, node.arg1) + ret = ASMap() + if binnode.ins != _Instruction.END: + #pylint: disable=protected-access + ret._set_trie(recurse(binnode, 0)) + return ret + + def to_binary(self, fill: bool = False) -> bytes: + """ + Convert this ASMap object to binary. + + Argument: + fill: permit the resulting binary encoder to contain mappers for + unassigned subnets in this ASMap object. Doing so may + reduce the size of the encoding. + Returns: + A bytes object with the encoding of this ASMap object. + """ + bits: List[int] = [] + + def recurse(node: _BinNode) -> None: + _CODER_INS.encode(node.ins.value, bits) + if node.ins == _Instruction.RETURN: + _CODER_ASN.encode(node.arg1, bits) + elif node.ins == _Instruction.JUMP: + _CODER_JUMP.encode(node.arg1.size, bits) + recurse(node.arg1) + recurse(node.arg2) + elif node.ins == _Instruction.DEFAULT: + _CODER_ASN.encode(node.arg1, bits) + recurse(node.arg2) + else: + assert node.ins == _Instruction.MATCH + _CODER_MATCH.encode(node.arg1, bits) + recurse(node.arg2) + + binnode = self._to_binnode(fill) + if binnode.ins != _Instruction.END: + recurse(binnode) + + val = 0 + nbits = 0 + ret = [] + for bit in bits: + val += (bit << nbits) + nbits += 1 + if nbits == 8: + ret.append(val) + val = 0 + nbits = 0 + if nbits: + ret.append(val) + return bytes(ret) + + @staticmethod + def from_binary(bindata: bytes) -> Optional["ASMap"]: + """Decode an ASMap object from the provided binary encoding.""" + + bits: List[int] = [] + for byte in bindata: + bits.extend((byte >> i) & 1 for i in range(8)) + + def recurse(bitpos: int) -> Tuple[_BinNode, int]: + insval, bitpos = _CODER_INS.decode(bits, bitpos) + ins = _Instruction(insval) + if ins == _Instruction.RETURN: + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + return _BinNode(ins, asn), bitpos + if ins == _Instruction.JUMP: + jump, bitpos = _CODER_JUMP.decode(bits, bitpos) + left, bitpos1 = recurse(bitpos) + if bitpos1 != bitpos + jump: + raise ValueError("Inconsistent jump") + right, bitpos = recurse(bitpos1) + return _BinNode(ins, left, right), bitpos + if ins == _Instruction.MATCH: + match, bitpos = _CODER_MATCH.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, match, sub), bitpos + assert ins == _Instruction.DEFAULT + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, asn, sub), bitpos + + if len(bits) == 0: + binnode = _BinNode(_Instruction.END) + else: + try: + binnode, bitpos = recurse(0) + except (ValueError, IndexError): + return None + if bitpos < len(bits) - 7: + return None + if not all(bit == 0 for bit in bits[bitpos:]): + return None + + return ASMap._from_binnode(binnode) + + def __lt__(self, other: "ASMap") -> bool: + return self._trie < other._trie + + def __eq__(self, other: object) -> bool: + if isinstance(other, ASMap): + return self._trie == other._trie + return False + + def extends(self, req: "ASMap") -> bool: + """Determine whether this matches req for all subranges where req is assigned.""" + def recurse(actual: List, require: List) -> bool: + if len(require) == 1 and require[0] == 0: + return True + if len(require) == 1: + if len(actual) == 1: + return bool(require[0] == actual[0]) + return recurse(actual[0], require) and recurse(actual[1], require) + if len(actual) == 2: + return recurse(actual[0], require[0]) and recurse(actual[1], require[1]) + return recurse(actual, require[0]) and recurse(actual, require[1]) + assert isinstance(req, ASMap) + #pylint: disable=protected-access + return recurse(self._trie, req._trie) + + def diff(self, other: "ASMap") -> List[ASNDiff]: + """Compute the diff from self to other.""" + prefix: List[bool] = [] + ret: List[ASNDiff] = [] + + def recurse(old_node: List, new_node: List): + if len(old_node) == 1 and len(new_node) == 1: + if old_node[0] != new_node[0]: + ret.append((list(prefix), old_node[0], new_node[0])) + else: + old_left: List = old_node if len(old_node) == 1 else old_node[0] + old_right: List = old_node if len(old_node) == 1 else old_node[1] + new_left: List = new_node if len(new_node) == 1 else new_node[0] + new_right: List = new_node if len(new_node) == 1 else new_node[1] + prefix.append(False) + recurse(old_left, new_left) + prefix[-1] = True + recurse(old_right, new_right) + prefix.pop() + assert isinstance(other, ASMap) + #pylint: disable=protected-access + recurse(self._trie, other._trie) + return ret + + def __copy__(self) -> "ASMap": + """Construct a copy of this ASMap object. Its state will not be shared.""" + ret = ASMap() + #pylint: disable=protected-access + ret._set_trie(copy.deepcopy(self._trie)) + return ret + + def __deepcopy__(self, _) -> "ASMap": + # ASMap objects do not allow sharing of the _trie member, so we don't need the memoization. + return self.__copy__() + + +class TestASMap(unittest.TestCase): + """Unit tests for this module.""" + + def test_ipv6_prefix_roundtrips(self) -> None: + """Test that random IPv6 network ranges roundtrip through prefix encoding.""" + for _ in range(20): + net_bits = random.getrandbits(128) + for prefix_len in range(0, 129): + masked_bits = (net_bits >> (128 - prefix_len)) << (128 - prefix_len) + net = ipaddress.IPv6Network((masked_bits.to_bytes(16, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_ipv4_prefix_roundtrips(self) -> None: + """Test that random IPv4 network ranges roundtrip through prefix encoding.""" + for _ in range(100): + net_bits = random.getrandbits(32) + for prefix_len in range(0, 33): + masked_bits = (net_bits >> (32 - prefix_len)) << (32 - prefix_len) + net = ipaddress.IPv4Network((masked_bits.to_bytes(4, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(32 <= len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_asmap_roundtrips(self) -> None: + """Test case that verifies random ASMap objects roundtrip to/from entries/binary.""" + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 24): + # Iterate over the probability that leaves are unassigned. + for pct in range(101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Run tests for to_entries and construction from those entries, both + # for overlapping and non-overlapping ones. + for overlapping in [False, True]: + entries = asmap.to_entries(overlapping=overlapping, fill=False) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertEqual(asmap2, asmap) + entries = asmap.to_entries(overlapping=overlapping, fill=True) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertTrue(asmap2.extends(asmap)) + + # Run tests for to_binary and construction from binary. + enc = asmap.to_binary(fill=False) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertEqual(asmap3, asmap) + enc = asmap.to_binary(fill=True) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertTrue(asmap3.extends(asmap)) + + def test_patching(self) -> None: + """Test behavior of update, lookup, extends, and diff.""" + #pylint: disable=too-many-locals,too-many-nested-blocks + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 10): + # Iterate over the probability that leaves are unassigned. + for pct in range(0, 101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Make a copy of that asmap object to which patches will be applied. + # It starts off being equal to asmap. + patched = copy.copy(asmap) + # Keep a list of patches performed. + patches: List[ASNEntry] = [] + # Initially there cannot be any difference. + self.assertEqual(asmap.diff(patched), []) + # Make 5 patches, each building on top of the previous ones. + for _ in range(0, 5): + # Construct a random path and new ASN to assign it to, apply it to patched, + # and remember it in patches. + pathlen = random.randrange(5) + path = [random.getrandbits(1) != 0 for _ in range(pathlen)] + newasn = random.randrange(1 + (1 << asnbits)) + patched.update(path, newasn) + patches = [(path, newasn)] + patches + + # Compute the diff, and whether asmap extends patched, and the other way + # around. + diff = asmap.diff(patched) + self.assertEqual(asmap == patched, len(diff) == 0) + extends = asmap.extends(patched) + back_extends = patched.extends(asmap) + # Determine whether those extends results are consistent with the diff + # result. + self.assertEqual(extends, all(d[2] == 0 for d in diff)) + self.assertEqual(back_extends, all(d[1] == 0 for d in diff)) + # For every diff found: + for path, old_asn, new_asn in diff: + # Verify asmap and patched actually differ there. + self.assertTrue(old_asn != new_asn) + self.assertEqual(asmap.lookup(path), old_asn) + self.assertEqual(patched.lookup(path), new_asn) + for _ in range(2): + # Extend the path far enough that it's smaller than any mapped + # range, and check the lookup holds there too. + spec_path = list(path) + while len(spec_path) < 32: + spec_path.append(random.getrandbits(1) != 0) + self.assertEqual(asmap.lookup(spec_path), old_asn) + self.assertEqual(patched.lookup(spec_path), new_asn) + # Search through the list of performed patches to find the last one + # applying to the extended path (note that patches is in reverse + # order, so the first match should work). + found = False + for patch_path, patch_asn in patches: + if spec_path[:len(patch_path)] == patch_path: + # When found, it must match whatever the result was patched + # to. + self.assertEqual(new_asn, patch_asn) + found = True + break + # And such a patch must exist. + self.assertTrue(found) + +if __name__ == '__main__': + unittest.main() diff --git a/bottleneck.py b/bottleneck.py new file mode 100644 index 0000000..de5fb85 --- /dev/null +++ b/bottleneck.py @@ -0,0 +1,147 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +import ipaddress +import sys +import copy +import re +import itertools + +LINE_PATTERN = re.compile(r"^TABLE_DUMP2?\|[0-9]+\|B\|[0-9a-f:.]+\|[0-9]+\|([0-9a-f:./]+)\|([0-9, {}]*)\|") +LINE_AP_PATTERN = re.compile(r"^TABLE_DUMP2_AP\|[0-9]+\|B\|[0-9a-f:.]+\|[0-9]+\|([0-9a-f:./]+)\|(?:[0-9]+)\|([0-9, {}]*)\|") +SPACE_PATTERN = re.compile(r" +") + +def merge_path(ret, net, path, cnt): + assert len(path) > 0 + asn = path[-1] + if net not in ret: + ret[net] = {asn: (path, 1)} + cnt[0] += 1 + else: + old_paths = ret[net] + if asn not in old_paths: + old_paths[asn] = (path, 1) + cnt[0] += 1 + else: + old_path, old_count = old_paths[asn] + common_len = 0 + while common_len < len(old_path) and common_len < len(path): + if old_path[-common_len - 1] == path[-common_len - 1]: + common_len += 1 + else: + break + old_paths[asn] = (old_path[-common_len:], old_count + 1) + return True + +def valid_asn(asn, net): + if asn == 0 or asn == 65535 or (asn >= 65552 and asn <= 131072) or asn == 4294967295: +# print("Skipping reserved AS%i (RFC1930) for network %s" % (asn, net)) + pass + elif asn == 23456: +# print("Skipping transition AS%i (RFC6793) for network %s" % (asn, net)) + pass + elif (asn >= 64496 and asn <= 64511) or (asn >= 65536 and asn <= 65551): +# print("Skipping documentation AS%i (RFC4893,RFC5398) for network %s" % (asn, net)) + pass + elif (asn >= 64512 and asn <= 65534) or (asn >= 4200000000 and asn <= 4294967294): +# print("Skipping private AS%i (RFC5398,RFC6996) for network %s" % (asn, net)) + pass + else: + return True + return False + +def accept_net(net): + if net.is_multicast: + print("Skipping multicast network %s" % net) + pass + elif net.is_private: +# print("Skipping private network %s" % net) + pass + elif net.is_unspecified: +# print("Skipping unspecified network %s" % net) + pass + elif net.is_reserved: +# print("Skipping reserved network %s" % net) + pass + elif net.is_loopback: + print("Skipping loopback network %s" % net) + elif net.is_link_local: + print("Skipping link-local network %s" % net) + elif not net.is_global: +# print("Skipping non-global network %s" % net) + pass + elif net.prefixlen == 0: +# print("Skipping entire network %s" % net) + pass + elif net.prefixlen > 48 and isinstance(net, ipaddress.IPv6Network): +# print("Skipping IPv6 range smaller than a /48: %s" % net) + pass + elif net.prefixlen > 24 and isinstance(net, ipaddress.IPv4Network): +# print("Skipping IPv4 range smaller than a /24: %s" % net) + pass + else: + return True + return False + +routes = 0 +valid_routes = 0 +kept_routes = [0] +ASNS = set() +RES = {} +reports = 0 +for line in sys.stdin: + match = LINE_PATTERN.match(line) + if not match: + match = LINE_AP_PATTERN.match(line) + if match: + routes += 1 + try: + net = ipaddress.ip_network(match[1], strict=True) + except ValueError: + net = None + if net is None: + raise ValueError("Cannot parse network %s" % match[1]) + if accept_net(net): + path_str = match[2] + # Ignore AS_SETs at the end of the path. + while path_str.endswith('}'): + path_str = path_str[:path_str.rindex('{')] + # Drop anything up to and including the last AS_SET in the path before that. + close_pos = path_str.rfind('}') + if close_pos >= 0: + path_str = path_str[close_pos + 1:] + # Convert to list of AS numbers + as_path = [int(x) for x in path_str.split(' ') if len(x) > 0] + if len(as_path): + # Remove duplicates from the list. + as_path = [i[0] for i in itertools.groupby(as_path)] + if valid_asn(as_path[-1], net): + ASNS.add(as_path[-1]) + merge_path(RES, net, as_path, kept_routes) + valid_routes += 1 + else: + print("Skipping empty path for %s: %s" % (net, match[2])) + else: + print("Ignoring unparseable line: %s" % line.strip()) + if routes >= (reports + 1) * 10000000: + print("%i routes, %i valid, %i kept; %i ASNs; %i prefixes" % (valid_routes, routes, kept_routes[0], len(ASNS), len(RES))) + reports = routes // 10000000 + +with open("output-bottlenext.txt", "w") as out: + for net in sorted(RES, key=ipaddress.get_mixed_type_key): + paths = RES[net] + first = True + for path, cnt in sorted(paths.values(), key=lambda f: (-f[1], f[0])): + out.write("%s%s AS%i # %i times %s\n" % ("" if first else "# ", net, path[0], cnt, " ".join("AS%i" % x for x in path))) + first = False + out.write("\n") + +with open("output-final.txt", "w") as out: + for net in sorted(RES, key=ipaddress.get_mixed_type_key): + paths = RES[net] + first = True + for path, cnt in sorted(paths.values(), key=lambda f: (-f[1], f[0])): + out.write("%s%s AS%i # %i times %s\n" % ("" if first else "# ", net, path[-1], cnt, " ".join("AS%i" % x for x in path))) + first = False + out.write("\n") diff --git a/construct/construct.py b/construct/construct.py new file mode 100644 index 0000000..a33e8af --- /dev/null +++ b/construct/construct.py @@ -0,0 +1,85 @@ +import datetime +import urllib.request +import shutil +import tempfile +import os +import os.path + +FILES = { + "routeviews.bz2": "http://archive.routeviews.org/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-3.bz2": "http://archive.routeviews.org/route-views3/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-4.bz2": "http://archive.routeviews.org/route-views4/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-5.bz2": "http://archive.routeviews.org/route-views5/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-6.bz2": "http://archive.routeviews.org/route-views6/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-amsix.bz2": "http://archive.routeviews.org/route-views.amsix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-chicago.bz2": "http://archive.routeviews.org/route-views.chicago/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-chile.bz2": "http://archive.routeviews.org/route-views.chile/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-eqix.bz2": "http://archive.routeviews.org/route-views.eqix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-flix.bz2": "http://archive.routeviews.org/route-views.flix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-gorex.bz2": "http://archive.routeviews.org/route-views.gorex/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-kixp.bz2": "http://archive.routeviews.org/route-views.kixp/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-linx.bz2": "http://archive.routeviews.org/route-views.linx/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-napafrica.bz2": "http://archive.routeviews.org/route-views.napafrica/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-nwax.bz2": "http://archive.routeviews.org/route-views.nwax/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-phoix.bz2": "http://archive.routeviews.org/route-views.phoix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-telxatl.bz2": "http://archive.routeviews.org/route-views.telxatl/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-wide.bz2": "http://archive.routeviews.org/route-views.wide/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-sydney.bz2": "http://archive.routeviews.org/route-views.sydney/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-saopaulo.bz2": "http://archive.routeviews.org/route-views.saopaulo/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-saopaulo-2.bz2": "http://archive.routeviews.org/route-views2.saopaulo/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-sg.bz2": "http://archive.routeviews.org/route-views.sg/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-perth.bz2": "http://archive.routeviews.org/route-views.perth/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-sfmix.bz2": "http://archive.routeviews.org/route-views.sfmix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-soxrs.bz2": "http://archive.routeviews.org/route-views.soxrs/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-mwix.bz2": "http://archive.routeviews.org/route-views.mwix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-rio.bz2": "http://archive.routeviews.org/route-views.rio/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-fortaleza.bz2": "http://archive.routeviews.org/route-views.fortaleza/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-gixa.bz2": "http://archive.routeviews.org/route-views.gixa/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-bdix.bz2": "http://archive.routeviews.org/route-views.bdix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-bknix.bz2": "http://archive.routeviews.org/route-views.bknix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-uaeix.bz2": "http://archive.routeviews.org/route-views.uaeix/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "routeviews-ny.bz2": "http://archive.routeviews.org/route-views.ny/bgpdata/%Y.%m/RIBS/rib.%Y%m%d.0000.bz2", + "ripe-00.gz": "https://data.ris.ripe.net/rrc00/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-01.gz": "https://data.ris.ripe.net/rrc01/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-03.gz": "https://data.ris.ripe.net/rrc03/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-04.gz": "https://data.ris.ripe.net/rrc04/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-05.gz": "https://data.ris.ripe.net/rrc05/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-06.gz": "https://data.ris.ripe.net/rrc06/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-07.gz": "https://data.ris.ripe.net/rrc07/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-10.gz": "https://data.ris.ripe.net/rrc10/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-11.gz": "https://data.ris.ripe.net/rrc11/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-12.gz": "https://data.ris.ripe.net/rrc12/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-13.gz": "https://data.ris.ripe.net/rrc13/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-14.gz": "https://data.ris.ripe.net/rrc14/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-15.gz": "https://data.ris.ripe.net/rrc15/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-16.gz": "https://data.ris.ripe.net/rrc16/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-18.gz": "https://data.ris.ripe.net/rrc18/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-19.gz": "https://data.ris.ripe.net/rrc19/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-20.gz": "https://data.ris.ripe.net/rrc20/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-21.gz": "https://data.ris.ripe.net/rrc21/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-22.gz": "https://data.ris.ripe.net/rrc22/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-23.gz": "https://data.ris.ripe.net/rrc23/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-24.gz": "https://data.ris.ripe.net/rrc24/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-25.gz": "https://data.ris.ripe.net/rrc25/%Y.%m/bview.%Y%m%d.0000.gz", + "ripe-26.gz": "https://data.ris.ripe.net/rrc26/%Y.%m/bview.%Y%m%d.0000.gz", +} + +NOW = datetime.datetime.now() - datetime.timedelta(seconds=300000) +NOWSTR = NOW.strftime("%Y%m%d") +DIRNAME = "data-%s/" % NOWSTR + +if not os.path.exists(DIRNAME): + os.mkdir(DIRNAME) +for filename, url in FILES.items(): + fullpath = DIRNAME + filename + eurl = NOW.strftime(url) + if not os.path.exists(fullpath): + print("Downloading %s from %s" % (filename, eurl)) + if os.path.exists(fullpath + ".part"): + os.remove(fullpath + ".part") + try: + with urllib.request.urlopen(eurl) as response, open(fullpath + ".part", "wb") as out_file: + shutil.copyfileobj(response, out_file) + os.rename(fullpath + ".part", fullpath) + except OSError as err: + print("Failed to download %s: %s" % (filename, err)) diff --git a/remote_dumps/prepare.sh b/remote_dumps/prepare.sh index 76d48f4..450fca5 100644 --- a/remote_dumps/prepare.sh +++ b/remote_dumps/prepare.sh @@ -1,5 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash rm dumps/* rm paths/* -rm prefix_asns.out \ No newline at end of file +rm prefix_asns.out diff --git a/remote_dumps/quagga_parse.sh b/remote_dumps/quagga_parse.sh index 5e63db2..16e8b92 100644 --- a/remote_dumps/quagga_parse.sh +++ b/remote_dumps/quagga_parse.sh @@ -1,8 +1,8 @@ -#!/bin/bash +#!/usr/bin/env bash for mrt in `ls dumps`; do - /bin/echo -n "processing $mrt... " + echo "processing $mrt..." OUT=$mrt - /usr/local/bin/bgpdump -vm dumps/$mrt | cut -d '|' -f '6,7' > paths/$OUT + /usr/bin/env bgpdump -vm dumps/$mrt | cut -d '|' -f '6,7' > paths/$OUT done diff --git a/remote_dumps/setup.sh b/remote_dumps/setup.sh index 5edb7d6..536447f 100644 --- a/remote_dumps/setup.sh +++ b/remote_dumps/setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash mkdir dumps mkdir paths @@ -9,4 +9,4 @@ rm libbgpdump-1.6.0.tgz cd libbgpdump-1.6.0 ./bootstrap.sh make install -cd .. \ No newline at end of file +cd ..