'Default' instructions

birdparse.py

@@ -2,22 +2,14 @@ import sys
 import re
 import ipaddress
 
-MAX_ASN = 1000000
-
 IPV4_PREFIX = bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff])
 
 def AddEntry(netmask, asn, fnam, linenum, entries):
     loc = "%s:%i" % (fnam, linenum)
+    network = ipaddress.ip_network(netmask, True)
     if asn is None:
         print("[WARNING] %s: no ASN for %s" % (loc, netmask), file=sys.stderr)
         return
-    if asn == 0:
-        print("[WARNING] %s: ASN is zero for %s" % (loc, netmask), file=sys.stderr)
-        return
-    if asn >= MAX_ASN:
-        print("[WARNING] %s: %s has too large AS%i" % (loc, netmask, asn), file=sys.stderr)
-        return
-    network = ipaddress.ip_network(netmask, True)
     if not network:
         print("[WARNING] %s: cannot parse netmask %s for AS%i" % (loc, netmask, asn), file=sys.stderr)
         return
@@ -36,6 +28,18 @@ def AddEntry(netmask, asn, fnam, linenum, entries):
     if network.is_loopback:
         print("[WARNING] %s: loopback address %s for AS%i" % (loc, netmask, asn), file=sys.stderr)
         return
+    if asn == 0 or asn == 65535 or (asn >= 65552 and asn <= 131072) or asn == 4294967295:
+        print("[WARNING] %s: prefix %s has reserved AS%i (RFC1930)" % (loc, netmask, asn), file=sys.stderr)
+        return
+    if asn == 23456:
+        print("[WARNING] %s: prefix %s has transition AS%i (RFC6793)" % (loc, netmask, asn), file=sys.stderr)
+        return
+    if (asn >= 64496 and asn <= 64511) or (asn >= 65536 and asn <= 65551):
+        print("[WARNING] %s: prefix %s has documentation AS%i (RFC4893,RFC5398)" % (loc, netmask, asn), file=sys.stderr)
+        return
+    if (asn >= 64512 and asn <= 65534) or (asn >= 4200000000 and asn <= 4294967294):
+        print("[WARNING] %s: prefix %s has private AS%i (RFC5398,RFC6996)" % (loc, netmask, asn), file=sys.stderr)
+        return
     if isinstance(network, ipaddress.IPv4Network):
         entries.append((IPV4_PREFIX + network.network_address.packed, "%s AS%i # %s:%i" % (network.compressed, asn, fnam, linenum)))
     elif isinstance(network, ipaddress.IPv6Network):

buildmap.py

@@ -4,7 +4,7 @@ import ipaddress
 
 def Parse(entries):
     for line in sys.stdin:
-        line = line.split('#')[0].rstrip(' \r\n')
+        line = line.split('#')[0].lstrip(' ').rstrip(' \r\n')
         prefix, asn = line.split(' ')
         assert(len(asn) > 2 and asn[:2] == "AS")
         network = ipaddress.ip_network(prefix)
@@ -48,66 +48,104 @@ def PrependPrefix(tree, bits):
             tree = [tree, None]
     return tree
 
-def CompactTree(tree):
+def CompactTree(tree, approx=True):
     num = 0
     if tree is None:
         return (tree, set())
     if isinstance(tree, int):
         return (tree, set([tree]))
-    tree[0], leftas = CompactTree(tree[0])
-    tree[1], rightas = CompactTree(tree[1])
+    tree[0], leftas = CompactTree(tree[0], approx)
+    tree[1], rightas = CompactTree(tree[1], approx)
     allas = leftas | rightas
     if len(allas) == 0:
         return (None, allas)
-    if len(allas) == 1:
+    if approx and len(allas) == 1:
         return (list(allas)[0], allas)
+    if isinstance(tree[0], int) and isinstance(tree[1], int) and tree[0] == tree[1]:
+        return tree[0], set([tree[0]])
     return (tree, allas)
 
+def DictMax(d):
+    mk = None
+    mv = None
+    for k, v in d.items():
+        if mv is None or v > mv:
+            mk, mv = k, v
+    return mk, mv
+ 
+def PropTree(tree, approx=True):
+    if tree is None:
+        return (tree, {}, True)
+    if isinstance(tree, int):
+        return (tree, {tree: 1}, False)
+    tree[0], leftcnt, leftnone = PropTree(tree[0], approx)
+    tree[1], rightcnt, rightnone = PropTree(tree[1], approx)
+    allcnt = {k: leftcnt.get(k, 0) + rightcnt.get(k, 0) for k in set(leftcnt) | set(rightcnt)}
+    allnone = leftnone | rightnone
+    maxasn, maxcount = DictMax(allcnt)
+    if maxcount is not None and maxcount >= 2 and (approx or not allnone):
+        return ([tree[0], tree[1], maxasn], {maxasn: 1}, allnone)
+    return (tree, allcnt, allnone)
+
 ZEROES = [0 for _ in range(129)]
 
-def TreeSize(tree, depth=0):
-    if tree is None:
-        return (0, 0, set())
+def TreeSize(tree, default = None):
+    if tree is None or tree == default:
+        return (0, 0, 0, set())
     if isinstance(tree, int):
-        return (1, 0, set([tree]))
-    left_as, left_node, left_set = TreeSize(tree[0], depth + 1)
-    right_as, right_node, right_set = TreeSize(tree[1], depth + 1)
-    return (left_as + right_as, left_node + right_node + 1, left_set | right_set)
+        return (1, 0, 0, set([tree]))
+    this_as = 0
+    this_set = set()
+    if len(tree) > 2 and tree[2] != default:
+        this_as = 1
+        default = tree[2]
+        this_set = set([default])
+    left_as, left_inas, left_node, left_set = TreeSize(tree[0], default)
+    right_as, right_inas, right_node, right_set = TreeSize(tree[1], default)
+    return (left_as + right_as, this_as + left_inas + right_inas, left_node + right_node + 1, left_set | right_set | this_set)
 
-def TreeSer(tree):
-    # 0: 3 byte ASN ollows
-    # 1: 4-byte ASN follows
+def TreeSer(tree, default):
+    # 0: 4-byte ASN ollows
+    # 1: 4-byte default ASN follows
     # 2-3: next bit is x
     # 4-7: next 2 bits are xx
     # 64-127: next 6 bits are xxxxxx
     # 128-131: N-byte jump offset follows
     # 132-239: jump offset 3-110
-    # 240-255: 2 byte ASN follows (with high 4 bits in header)
+    # 240-247: 2 byte ASN follows (with high 3 bits in header)
+    # 248-255: 2 byte default ASN follows (with high 3 bits in header)
     assert(tree is not None)
+    assert(not (isinstance(tree, int) and tree == default))
     bits = 0
     nbits = 0
     while nbits < 6 and isinstance(tree, list):
-        if tree[0] is None:
+        if tree[0] is None or tree[0] == default:
             bits = bits * 2 + 1
             nbits += 1
             tree = tree[1]
-        elif tree[1] is None:
+        elif tree[1] is None or tree[1] == default:
             bits = bits * 2
             nbits += 1
             tree = tree[0]
         else:
             break
     if nbits > 0:
-        return bytes([bits + (1 << nbits)]) + TreeSer(tree)
+        return bytes([bits + (1 << nbits)]) + TreeSer(tree, default)
     if isinstance(tree, int):
         asn = tree
-        if asn >= 2**24:
-            return bytes([1]) + asn.to_bytes(4, 'little')
-        if asn >= 2**20:
-            return bytes([0]) + asn.to_bytes(3, 'little')
+        if asn >= 2**19:
+            return bytes([0]) + asn.to_bytes(4, 'little')
         return bytes([240 + (asn >> 16), asn & 0xFF, (asn >> 8) & 0xFF])
-    left = TreeSer(tree[0])
-    right = TreeSer(tree[1])
+    newdef = bytes()
+    if len(tree) > 2 and tree[2] != default:
+        default = tree[2]
+        if default >= 2**19:
+            newdef = bytes([1]) + default.to_bytes(4, 'little')
+        else:
+            newdef = bytes([248 + (default >> 16), default & 0xFF, (default >> 8) & 0xFF])
+        return newdef + TreeSer(tree, default)
+    left = TreeSer(tree[0], default)
+    right = TreeSer(tree[1], default)
     leftlen = len(left)
     assert(leftlen >= 3)
     if leftlen <= 110:
@@ -117,21 +155,28 @@ def TreeSer(tree):
     assert(leftlennum <= 4)
     return bytes([127 + leftlennum]) + leftlen.to_bytes(leftlennum, 'little') + left + right
 
-def BuildTree(entries):
-    tree, _ = CompactTree(UpdateTree([None, None], 128, entries))
+def BuildTree(entries, approx=True):
+    tree = [None, None]
+    tree = UpdateTree(tree, 128, entries)
     return tree
 
 entries = []
-print("[INFO] Loading")
+print("[INFO] Loading", file=sys.stderr)
 Parse(entries)
 print("[INFO] Read %i prefixes" % len(entries), file=sys.stderr)
 print("[INFO] Constructing trie", file=sys.stderr)
 tree = BuildTree(entries)
-as_count, node_count, as_set = TreeSize(tree)
-print("[INFO] Number of prefixes: %i" % as_count, file=sys.stderr)
-print("[INFO] Number of distinct AS values: %i" % len(as_set), file=sys.stderr)
-print("[INFO] Number of decision nodes: %i" % node_count, file=sys.stderr)
-bs = TreeSer(tree)
+as_count, inas_count, node_count, as_set = TreeSize(tree)
+print("[INFO] Trie stats: %i inner, %i prefixes (%i leaf), %i distinct AS" % (node_count, as_count + inas_count, as_count, len(as_set)), file=sys.stderr)
+print("[INFO] Compacting tree", file=sys.stderr)
+tree, _ = CompactTree(tree, True)
+as_count, inas_count, node_count, as_set = TreeSize(tree)
+print("[INFO] Trie stats: %i inner, %i prefixes (%i leaf), %i distinct AS" % (node_count, as_count + inas_count, as_count, len(as_set)), file=sys.stderr)
+print("[INFO] Computing inner prefixes", file=sys.stderr)
+tree, _, _ = PropTree(tree, True)
+as_count, inas_count, node_count, as_set = TreeSize(tree)
+print("[INFO] Trie stats: %i inner, %i prefixes (%i leaf), %i distinct AS" % (node_count, as_count + inas_count, as_count, len(as_set)), file=sys.stderr)
+bs = TreeSer(tree, None)
 print("[INFO] Serialized trie is %i bytes" % len(bs), file=sys.stderr)
 print("[INFO] Writing trie to stdout", file=sys.stderr)
 sys.stdout.buffer.write(bs)

dumpmap.py

@@ -10,31 +10,47 @@ def num_to_addr_str(num, bits):
     else:
         return "%s/%i" % (ipaddress.IPv6Address(num).compressed, 128 - bits)
 
-def dumpmap(asmap, pos=0, num=0, bits=128):
+def dumpmap(asmap, pos, num, bits, level, default):
     assert(len(asmap) >= pos + 1)
-    opcode = int(asmap[pos])
+    opcode = 0 + asmap[pos]
     pos += 1
-    if opcode < 2:
-        assert(len(asmap) >= pos + 2 + opcode)
-        print("%s AS%i" % (num_to_addr_str(num, bits), int.from_bytes(asmap[pos:pos+2+opcode], 'little')))
-    elif opcode >= 240:
+    if opcode == 0:
+        assert(len(asmap) >= pos + 4 + opcode)
+        asn = int.from_bytes(asmap[pos:pos+4+opcode], 'little')
+        assert(asn != default)
+        print("%s%s AS%i" % ("  " * level, num_to_addr_str(num, bits), asn))
+    elif opcode == 1:
+        assert(len(asmap) >= pos + 4 + opcode)
+        asn = int.from_bytes(asmap[pos:pos+4+opcode], 'little')
+        assert(asn != default)
+        print("%s%s AS%i" % ("  " * level, num_to_addr_str(num, bits), asn))
+        dumpmap(asmap, pos+4, num, bits, level+1, asn)
+    elif opcode >= 240 and opcode < 248:
         assert(len(asmap) >= pos + 2)
-        print("%s AS%i" % (num_to_addr_str(num, bits), int.from_bytes(asmap[pos:pos+2], 'little') + ((opcode - 240) << 16)))
+        asn = int.from_bytes(asmap[pos:pos+2], 'little') + ((opcode - 240) << 16)
+#        assert(asn != default)
+        print("%s%s AS%i" % ("  " * level, num_to_addr_str(num, bits), asn))
+    elif opcode >= 248:
+        assert(len(asmap) >= pos + 2)
+        asn = int.from_bytes(asmap[pos:pos+2], 'little') + ((opcode - 248) << 16)
+#        assert(asn != default)
+        print("%s%s AS%i" % ("  " * level, num_to_addr_str(num, bits), asn))
+        dumpmap(asmap, pos+2, num, bits, level+1, asn)
     elif opcode < 128:
         nbits = opcode.bit_length() - 1
-        dumpmap(asmap, pos, (num << nbits) + (opcode & ((1 << nbits) - 1)), bits - nbits)
+        dumpmap(asmap, pos, (num << nbits) + (opcode & ((1 << nbits) - 1)), bits - nbits, level, default)
     elif opcode >= 132:
-        dumpmap(asmap, pos, num * 2, bits - 1)
-        dumpmap(asmap, pos + opcode - 129, num * 2 + 1, bits - 1)
+        dumpmap(asmap, pos, num * 2, bits - 1, level, default)
+        dumpmap(asmap, pos + opcode - 129, num * 2 + 1, bits - 1, level, default)
     else:
         offlen = opcode - 127
         assert(len(asmap) >= pos + offlen)
         offset = int.from_bytes(asmap[pos:pos+offlen], 'little')
         pos += offlen
-        dumpmap(asmap, pos, num * 2, bits - 1)
-        dumpmap(asmap, pos + offset, num * 2 + 1, bits - 1)
+        dumpmap(asmap, pos, num * 2, bits - 1, level, default)
+        dumpmap(asmap, pos + offset, num * 2 + 1, bits - 1, level, default)
 
 with open(sys.argv[1], "rb") as f:
     asmap = f.read()
 
-dumpmap(asmap)
+dumpmap(asmap, 0, 0, 128, 0, None)

testmap.py

@@ -1,25 +1,32 @@
 import sys
 import ipaddress
 
-IPV4_PREFIX = bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff])
-
 def interpret(asmap, num, bits):
     pos = 0
+    default = None
     while True:
         assert(len(asmap) >= pos + 1)
         opcode = int(asmap[pos])
         pos += 1
-        if opcode < 2:
-            assert(len(asmap) >= pos + 2 + opcode)
-            return int.from_bytes(asmap[pos:pos+2+opcode], 'little')
-        elif opcode >= 240:
+        if opcode == 0:
+            assert(len(asmap) >= pos + 4 + opcode)
+            return int.from_bytes(asmap[pos:pos+4+opcode], 'little')
+        elif opcode == 1:
+            assert(len(asmap) >= pos + 4 + opcode)
+            default = int.from_bytes(asmap[pos:pos+4+opcode], 'little')
+            pos += 4
+        elif opcode >= 240 and opcode < 248:
             assert(len(asmap) >= pos + 2)
             return int.from_bytes(asmap[pos:pos+2], 'little') + ((opcode - 240) << 16)
+        elif opcode >= 248:
+            assert(len(asmap) >= pos + 2)
+            default = int.from_bytes(asmap[pos:pos+2], 'little') + ((opcode - 248) << 16)
+            pos += 2
         elif opcode < 128:
             nbits = opcode.bit_length() - 1
             assert(bits >= nbits)
             if (opcode ^ (num >> (bits - nbits))) & ((1 << nbits) - 1):
-                return None
+                return default
             bits -= nbits
         elif opcode >= 132:
             assert(bits >= 1)
@@ -40,7 +47,7 @@ with open(sys.argv[1], "rb") as f:
     asmap = f.read()
 addr = ipaddress.ip_address(sys.argv[2])
 if isinstance(addr, ipaddress.IPv4Address):
-    num = int.from_bytes(IPV4_PREFIX + addr.packed, 'big')
+    num = int.from_bytes(addr.packed, 'big') + 0xffff00000000
 elif isinstance(addr, ipaddress.IPv6Address):
     num = int.from_bytes(addr.packed, 'big')