Improve bgpmap png

lg.cfg

@@ -6,4 +6,15 @@ PROXY = {
 		"h3": 5000,
 	}
 
+# Used for bgpmap
+ROUTER_IP = {
+        "gw" : [ "91.224.148.2", "2a01:6600:8000::175" ],
+        "h3" : [ "91.224.148.3", "2a01:6600:8000::131" ]
+}
+
+AS_NUMBER = {
+    "gw" : "197422",
+    "h3" : "197422"
+}
+
 SESSION_KEY = '\xd77\xf9\xfa\xc2\xb5\xcd\x85)`+H\x9d\xeeW\\%\xbe/\xbaT\x89\xe8\xa7'

lg.py

@@ -34,41 +34,35 @@ from flask import Flask, render_template, jsonify, redirect, session, request, a
 app = Flask(__name__)
 app.config.from_pyfile('lg.cfg')
 
-def add_links(text):
-	"""Browser a string and replace ipv4, ipv6, as number, with a whois link """
 
-	if type(text) in [ str, unicode ]:
+def add_links(text):
+    """Browser a string and replace ipv4, ipv6, as number, with a
+    whois link """
+
+    if type(text) in [str, unicode]:
         text = text.split("\n")
 
     ret_text = []
     for line in text:
         # Some heuristic to create link
         if line.strip().startswith("BGP.as_path:") or \
-			line.strip().startswith("Neighbor AS:") :
+            line.strip().startswith("Neighbor AS:"):
-			ret_text.append(re.sub(r'(\d+)',r'<a href="/whois/\1" class="whois">\1</a>',line))
+            ret_text.append(re.sub(r'(\d+)', r'<a href="/whois/\1" class="whois">\1</a>', line))
         else:
-			line = re.sub(r'([a-zA-Z0-9\-]*\.([a-zA-Z]{2,3}){1,2})(\s|$)', r'<a href="/whois/\1" class="whois">\1</a>\3',line)
+            line = re.sub(r'([a-zA-Z0-9\-]*\.([a-zA-Z]{2,3}){1,2})(\s|$)', r'<a href="/whois/\1" class="whois">\1</a>\3', line)
-			line = re.sub(r'AS(\d+)', r'<a href="/whois/\1" class="whois">AS\1</a>',line)
+            line = re.sub(r'AS(\d+)', r'<a href="/whois/\1" class="whois">AS\1</a>', line)
-			line = re.sub(r'(\d+\.\d+\.\d+\.\d+)', r'<a href="/whois/\1" class="whois">\1</a>',line)
+            line = re.sub(r'(\d+\.\d+\.\d+\.\d+)', r'<a href="/whois/\1" class="whois">\1</a>', line)
             hosts = "/".join(request.path.split("/")[2:])
             line = re.sub(r'\[(\w+)\s+((|\d\d\d\d-\d\d-\d\d\s)(|\d\d:)\d\d:\d\d|\w\w\w\d\d)', r'[<a href="/detail/%s?q=\1">\1</a> \2' % hosts, line)
-			line = re.sub(r'(^|\s+)(([a-f\d]{0,4}:){3,10}[a-f\d]{0,4})', r'\1<a href="/whois/\2" class="whois">\2</a>',line , re.I)
+            line = re.sub(r'(^|\s+)(([a-f\d]{0,4}:){3,10}[a-f\d]{0,4})', r'\1<a href="/whois/\2" class="whois">\2</a>', line, re.I)
             ret_text.append(line)
     return "\n".join(ret_text)
 
-def extract_paths(text):
-	paths = []
-	for line in text:
-		line = line.strip()
-		if line.startswith("BGP.as_path:"):
-			paths.append(line.replace("BGP.as_path:", "").strip().split(" "))
-
-	return paths
 
 def set_session(request_type, hosts, proto, request_args):
     """ Store all data from user in the user session """
     session.permanent = True
-	session.update( {
+    session.update({
         "request_type": request_type,
         "hosts": hosts,
         "proto": proto,
@@ -77,7 +71,8 @@ def set_session(request_type, hosts, proto, request_args):
     history = session.get("history", [])
 
     # erase old format history
-	if type(history) != type(list()): history = []
+    if type(history) != type(list()):
+        history = []
 
     t = (hosts, proto, request_type, request_args)
     if t in history:
@@ -85,13 +80,16 @@ def set_session(request_type, hosts, proto, request_args):
     history.insert(0, t)
     session["history"] = history[:20]
 
+
 def whois_command(query):
-	return subprocess.Popen( [ 'whois', query], stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore')
+    return subprocess.Popen(['whois', query], stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore')
+
 
 def bird_command(host, proto, query):
     """Alias to bird_proxy for bird service"""
     return bird_proxy(host, proto, "bird", query)
 
+
 def bird_proxy(host, proto, service, query):
     """Retreive data of a service from a running lg-proxy on a remote node
 
@@ -103,9 +101,13 @@ def bird_proxy(host, proto, service, query):
     """
 
     path = ""
-	if proto == "ipv6": path = service + "6"
+    if proto == "ipv6":
-	elif proto == "ipv4": path = service
+        path = service + "6"
-	port = app.config["PROXY"].get(host,"")
+    elif proto == "ipv4":
+        path = service
+
+    port = app.config["PROXY"].get(host, "")
+
     if not port or not path:
         return False, "Host/Proto not allowed"
     else:
@@ -119,57 +121,73 @@ def bird_proxy(host, proto, service, query):
             status = False
         return status, resultat
 
+
 @app.context_processor
 def inject_commands():
     commands = [
-			("traceroute","traceroute ..."),
+            ("traceroute", "traceroute ..."),
-			("summary","show protocols"),
+            ("summary", "show protocols"),
-			("detail","show protocols ... all"),
+            ("detail", "show protocols ... all"),
-			("prefix","show route for ..."),
+            ("prefix", "show route for ..."),
-			("prefix_detail","show route for ... all"),
+            ("prefix_detail", "show route for ... all"),
-			("where","show route where net ~ [ ... ]"),
+            ("prefix_bgpmap", "show route for ... (bgpmap)"),
-			("where_detail","show route where net ~ [ ... ] all"),
+            ("where", "show route where net ~ [ ... ]"),
-			("adv","show route ..."),
+            ("where_detail", "show route where net ~ [ ... ] all"),
+            ("where_bgpmap", "show route where net ~ [ ... ] (bgpmap)"),
+            ("adv", "show route ..."),
+            ("adv_bgpmap", "show route ... (bgpmap)"),
         ]
     commands_dict = {}
     for id, text in commands:
         commands_dict[id] = text
     return dict(commands=commands, commands_dict=commands_dict)
 
+
 @app.context_processor
 def inject_all_host():
     return dict(all_hosts="+".join(app.config["PROXY"].keys()))
 
+
 @app.route("/")
 def hello():
-	return redirect("/summary/%s/ipv4" % "+".join(app.config["PROXY"].keys()) )
+    return redirect("/summary/%s/ipv4" % "+".join(app.config["PROXY"].keys()))
+
 
 def error_page(text):
-	return render_template('error.html', error = text ), 500
+    return render_template('error.html', error=text), 500
+
 
 @app.errorhandler(400)
 def incorrect_request(e):
         return render_template('error.html', warning="The server could not understand the request"), 400
 
+
 @app.errorhandler(404)
 def page_not_found(e):
         return render_template('error.html', warning="The requested URL was not found on the server."), 404
 
+
 @app.route("/whois/<query>")
 def whois(query):
-	if not query.strip(): abort(400)
+    if not query.strip():
+        abort(400)
+
     try:
         asnum = int(query)
-		query = "as%d"%asnum
+        query = "as%d" % asnum
     except:
         m = re.match(r"[\w\d-]*\.(?P<domain>[\d\w-]+\.[\d\w-]+)$", query)
-		if m: query = query.groupdict()["domain"]
+        if m:
-	output = whois_command(query).replace("\n","<br>")
+            query = query.groupdict()["domain"]
+
+    output = whois_command(query).replace("\n", "<br>")
     return jsonify(output=output, title=query)
 
+
 SUMMARY_UNWANTED_PROTOS = ["Kernel", "Static", "Device"]
 SUMMARY_RE_MATCH = r"(?P<name>[\w_]+)\s+(?P<proto>\w+)\s+(?P<table>\w+)\s+(?P<state>\w+)\s+(?P<since>((|\d\d\d\d-\d\d-\d\d\s)(|\d\d:)\d\d:\d\d|\w\w\w\d\d))($|\s+(?P<info>.*))"
 
+
 @app.route("/summary/<hosts>")
 @app.route("/summary/<hosts>/<proto>")
 def summary(hosts, proto="ipv4"):
@@ -181,27 +199,29 @@ def summary(hosts, proto="ipv4"):
     for host in hosts.split("+"):
         ret, res = bird_command(host, proto, command)
         res = res.split("\n")
-		if len(res) > 1: #if ret:
+        if len(res) > 1:
             data = []
             for line in res[1:]:
                 line = line.strip()
-				if line and ( line.split() + [""] )[1] not in SUMMARY_UNWANTED_PROTOS:
+                if line and (line.split() + [""])[1] not in SUMMARY_UNWANTED_PROTOS:
-					m = re.match(SUMMARY_RE_MATCH ,line)
+                    m = re.match(SUMMARY_RE_MATCH, line)
                     if m:
                         data.append(m.groupdict())
                     else:
-						app.logger.warning("couldn't parse: %s" , line)
+                        app.logger.warning("couldn't parse: %s", line)
 
             summary[host] = data
         else:
-			error.append("%s: bird command failed with error, %s" % (host,"\n".join(res)))
+            error.append("%s: bird command failed with error, %s" % (host, "\n".join(res)))
 
     return render_template('summary.html', summary=summary, command=command, error="<br>".join(error))
 
+
 @app.route("/detail/<hosts>/<proto>")
 def detail(hosts, proto):
-	name = request.args.get('q', '')
+    name = request.args.get('q', '').strip()
-	if not name.strip(): abort(400)
+    if not name:
+        abort(400)
 
     set_session("detail", hosts, proto, name)
     command = "show protocols all %s" % name
@@ -211,25 +231,32 @@ def detail(hosts, proto):
     for host in hosts.split("+"):
         ret, res = bird_command(host, proto, command)
         res = res.split("\n")
-		if len(res) > 1 : #if ret:
+        if len(res) > 1:
-			detail[host] = { "status": res[1], "description": add_links(res[2:]) }
+            detail[host] = {"status": res[1], "description": add_links(res[2:])}
         else:
-			error.append("%s: bird command failed with error, %s" % (host,"\n".join(res)))
+            error.append("%s: bird command failed with error, %s" % (host, "\n".join(res)))
 
     return render_template('detail.html', detail=detail, command=command, error="<br>".join(error))
 
+
 @app.route("/traceroute/<hosts>/<proto>")
 def traceroute(hosts, proto):
-	q = request.args.get('q', '')
+    q = request.args.get('q', '').strip()
-	if not q.strip(): abort(400)
+    if not q:
+        abort(400)
+
     set_session("traceroute", hosts, proto, q)
 
     if proto == "ipv6" and not ipv6_is_valid(q):
-		try: q = resolve(q, "AAAA")
+        try:
-		except:	return error_page("%s is unresolvable or invalid for %s" % (q, proto))
+            q = resolve(q, "AAAA")
+        except:
+            return error_page("%s is unresolvable or invalid for %s" % (q, proto))
     if proto == "ipv4" and not ipv4_is_valid(q):
-		try: q = resolve(q, "A")
+        try:
-		except:	return error_page("%s is unresolvable or invalid for %s" % (q, proto))
+            q = resolve(q, "A")
+        except:
+            return error_page("%s is unresolvable or invalid for %s" % (q, proto))
 
     infos = {}
     for host in hosts.split("+"):
@@ -237,30 +264,61 @@ def traceroute(hosts, proto):
         infos[host] = add_links(resultat)
     return render_template('traceroute.html', infos=infos)
 
+
 @app.route("/adv/<hosts>/<proto>")
 def show_route_filter(hosts, proto):
     return show_route("adv", hosts, proto)
 
+
+@app.route("/adv_bgpmap/<hosts>/<proto>")
+def show_route_filter_bgpmap(hosts, proto):
+    return show_route("adv_bgpmap", hosts, proto)
+
+
 @app.route("/where/<hosts>/<proto>")
 def show_route_where(hosts, proto):
     return show_route("where", hosts, proto)
 
+
 @app.route("/where_detail/<hosts>/<proto>")
 def show_route_where_detail(hosts, proto):
     return show_route("where_detail", hosts, proto)
 
+
+@app.route("/where_bgpmap/<hosts>/<proto>")
+def show_route_where_bgpmap(hosts, proto):
+    return show_route("where_bgpmap", hosts, proto)
+
+
 @app.route("/prefix/<hosts>/<proto>")
 def show_route_for(hosts, proto):
     return show_route("prefix", hosts, proto)
 
+
 @app.route("/prefix_detail/<hosts>/<proto>")
 def show_route_for_detail(hosts, proto):
     return show_route("prefix_detail", hosts, proto)
 
+
+@app.route("/prefix_bgpmap/<hosts>/<proto>")
+def show_route_for_bgpmap(hosts, proto):
+    return show_route("prefix_bgpmap", hosts, proto)
+
+
 ASNAME_CACHE_FILE = "/tmp/asname_cache.pickle"
 ASNAME_CACHE = load_cache_pickle(ASNAME_CACHE_FILE, {})
+
 def get_as_name(_as):
-	if not ASNAME_CACHE.has_key(_as):
+    """return a string that contain the as number following by the as name
+
+    It's the use whois database informations
+    # Warning, the server can be blacklisted from ripe is too many requests are done
+    """
+
+    if not _as.isdigit():
+        return _as
+
+    if _as not in ASNAME_CACHE:
         whois_answer = whois_command("as%s" % _as)
         as_name = re.search('(as-name|ASName): (.*)', whois_answer)
         if as_name:
@@ -273,51 +331,144 @@ def get_as_name(_as):
     else:
         return "AS%s\r%s" % (_as, ASNAME_CACHE[_as])
 
-@app.route("/bgpmap/<data>")
+
-def show_bgpmap(data):
+@app.route("/bgpmap/")
+def show_bgpmap():
+    """return a bgp map in a png file, from the json tree in q argument"""
+
+    data = request.args.get('q', '').strip()
+    if not data:
+        abort(400)
+
     data = json.loads(unquote(data))
+
     graph = pydot.Dot('BGPMAP', graph_type='digraph')
+
     nodes = {}
     edges = {}
+
+    def add_node(_as, **kwargs):
+        if _as not in nodes:
+            kwargs["label"] = kwargs.get("label", get_as_name(_as))
+            nodes[_as] = pydot.Node(_as, style="filled", fontsize="10", **kwargs)
+            graph.add_node(nodes[_as])
+        return nodes[_as]
+
+    def add_edge(_previous_as, _as, **kwargs):
+        kwargs["splines"] = "true"
+        force = kwargs.get("force", False)
+
+        edge_tuple = (_previous_as, _as)
+        if force or edge_tuple not in edges:
+            edge = pydot.Edge(*edge_tuple, **kwargs)
+            graph.add_edge(edge)
+            edges[edge_tuple] = edge
+        return edges[edge_tuple]
+
     for host, asmaps in data.iteritems():
-		nodes[host] = pydot.Node(host, shape="box", style="filled", fillcolor="#F5A9A9")
+        add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
-		graph.add_node(nodes[host])
+
+        as_number = app.config["AS_NUMBER"].get(host, None)
+        if as_number:
+            node = add_node(as_number, fillcolor="#F5A9A9")
+            edge = add_edge(as_number, nodes[host])
+            edge.set_color("red")
+            edge.set_style("bold")
+    
+    colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93" ]
+    color_index = 0
+    hosts = data.keys()
+
     for host, asmaps in data.iteritems():
         first = True
         for asmap in asmaps:
             previous_as = host
+            color_index = color_index + 1
             for _as in asmap:
-				_as = get_as_name(_as)
                 if _as == previous_as:
                     continue
-				if not nodes.has_key(_as):
-					nodes[_as] = pydot.Node(_as, label=_as, style="filled", fillcolor=(first and "#F5A9A9" or "white"))
-					graph.add_node(nodes[_as])
 
-				edge_tuple = (nodes[previous_as], nodes[_as])
+                add_node(_as, fillcolor=(first and "#F5A9A9" or "white"))
-				if not edges.has_key(edge_tuple):
+                edge = add_edge(nodes[previous_as], nodes[_as] )
-					edge = pydot.Edge(*edge_tuple)
-					graph.add_edge(edge)
-					edges[edge_tuple] = edge
 
-				if edge.get_color() != "red" and first:
+                if first:
+                    edge.set_style("bold")
                     edge.set_color("red")
+                elif edge.get_color() != "red":
+                    edge.set_style("dashed")
+                    edge.set_color(colors[color_index])
 
                 previous_as = _as
             first = False
 
+    node = add_node(previous_as)
+    node.set_shape("box")
     #return Response("<pre>" + graph.create_dot() + "</pre>")
     return Response(graph.create_png(), mimetype='image/png')
 
+
+def build_as_tree_from_raw_bird_ouput(host, proto, text):
+    """Extract the as path from the raw bird "show route all" command"""
+
+    path = None
+    paths = []
+    net_dest = None
+    for line in text:
+        line = line.strip()
+
+        expr = re.search(r'(.*)via\s+([0-9:\.]+)\s+on.*\[(\w+)\s+', line)
+        if expr:
+            if path:
+                path.append(net_dest)
+                paths.append(path)
+                path = None
+
+            if expr.group(1).strip():
+                net_dest = expr.group(1).strip()
+
+            peer_ip = expr.group(2).strip()
+            peer_protocol_name = expr.group(3).strip()
+            # Check if via line is a internal route
+            for rt_host, rt_ips in app.config["ROUTER_IP"].iteritems():
+                if peer_ip in rt_ips:
+                    path = [rt_host]
+                    break
+            else:  # retreive as number from bird
+                ret, res = bird_command(host, proto, "show protocols all %s" % peer_protocol_name)
+                re_asnumber = re.search("Neighbor AS:\s*(\d*)", res)
+                if re_asnumber:
+                    path = [re_asnumber.group(1)]
+                else:
+                    print "Missing retreive some information for the path"
+                    path = ["as?????"]
+
+        if line.startswith("BGP.as_path:"):
+            path.extend(line.replace("BGP.as_path:", "").strip().split(" "))
+    
+    if path:
+        path.append(net_dest)
+        paths.append(path)
+
+    return paths
+
+
 def show_route(request_type, hosts, proto):
-	expression = unquote(request.args.get('q', ''))
+    expression = unquote(request.args.get('q', '')).strip()
-	if not expression.strip(): abort(400)
+    if not expression:
+        abort(400)
+
     set_session(request_type, hosts, proto, expression)
 
-	all = (request_type.endswith("detail") and " all" or "" )
+    bgpmap = request_type.endswith("bgpmap")
+
+    all = (request_type.endswith("detail") and " all" or "")
+    if bgpmap:
+        all = " all"
 
     if request_type.startswith("adv"):
-		command = "show route " + expression
+        command = "show route " + expression.strip()
+        if bgpmap and not command.endswith("all"):
+            command = command + " all"
     elif request_type.startswith("where"):
         command = "show route where net ~ [ " + expression + " ]" + all
     else:
@@ -325,34 +476,48 @@ def show_route(request_type, hosts, proto):
         if len(expression.split("/")) > 1:
             expression, mask = (expression.split("/"))
 
-		if not mask and proto == "ipv4" : mask = "32"
+        if not mask and proto == "ipv4":
-		if not mask and proto == "ipv6" : mask = "128"
+            mask = "32"
+        if not mask and proto == "ipv6":
+            mask = "128"
         if not mask_is_valid(mask):
             return error_page("mask %s is invalid" % mask)
-		if proto == "ipv6" and not ipv6_is_valid(expression):
-			try: expression = resolve(expression, "AAAA")
-			except:	return error_page("%s is unresolvable or invalid for %s" % (expression, proto))
-		if proto == "ipv4" and not ipv4_is_valid(expression):
-			try: expression = resolve(expression, "A")
-			except:	return error_page("%s is unresolvable or invalid for %s" % (expression, proto))
 
-		if mask: expression += "/" + mask
+        if proto == "ipv6" and not ipv6_is_valid(expression):
+            try:
+                expression = resolve(expression, "AAAA")
+            except:
+                return error_page("%s is unresolvable or invalid for %s" % (expression, proto))
+        if proto == "ipv4" and not ipv4_is_valid(expression):
+            try:
+                expression = resolve(expression, "A")
+            except:
+                return error_page("%s is unresolvable or invalid for %s" % (expression, proto))
+
+        if mask:
+            expression += "/" + mask
+
         command = "show route for " + expression + all
 
     detail = {}
     error = []
-	bgpmap = {}
     for host in hosts.split("+"):
         ret, res = bird_command(host, proto, command)
 
         res = res.split("\n")
-		if len(res) > 1 : #if ret:
+        if len(res) > 1:
-			detail[host] = add_links(res)
+            if bgpmap:
-			bgpmap[host] = extract_paths(res)
+                detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res)
             else:
-			error.append("%s: bird command failed with error, %s" % (host,"\n".join(res)))
+                detail[host] = add_links(res)
+        else:
+            error.append("%s: bird command failed with error, %s" % (host, "\n".join(res)))
+
+    if bgpmap:
+        detail = json.dumps(detail)
+
+    return render_template((bgpmap and 'bgpmap.html' or 'route.html'), detail=detail, command=command, expression=expression, error="<br>".join(error))
 
-	return render_template('route.html', detail=detail, command=command, expression=expression, bgpmap=json.dumps(bgpmap), error="<br>".join(error) )
 
 app.secret_key = app.config["SESSION_KEY"]
 app.debug = True

static/js/lg.js

@@ -80,7 +80,7 @@ $(function(){
 			reload();
 			});
 		$(".request_type ul a").click(function(){
-			if ( request_type.replace("_detail","") != $(this).attr('id').replace("_detail","") ){
+			if ( request_type.split("_")[0] != $(this).attr('id').split("_")[0] ){
 				request_args = ""
 				$(".request_args").val("");
 			}

templates/bgpmap.html

@@ -0,0 +1,9 @@
+{% extends "layout.html" %}
+{% block body %}
+<h3>{{session.hosts}}: {{command}}</h3>
+{% if session.request_args != expression|replace("/32","")|replace("/128","") %}
+<i>DNS: <a href="/whois/{{session.request_args}}" class="whois">{{session.request_args}}</a> => <a href="/whois/{{ expression|replace("/32","")|replace("/128","") }}" class="whois">{{expression|replace("/32","")|replace("/128","")}}</a></i><br />
+{% endif %}<br />
+<a href="/bgpmap/?q={{detail}}"><img src="/bgpmap/?q={{detail}}" /></a>
+<br />
+{% endblock %}

templates/route.html

@@ -1,7 +1,10 @@
 {% extends "layout.html" %}
 {% block body %}
 {% for host in detail %}
-<h3>{{host}}: {{command}}</h3>
+<h3>
+    {{host}}: {{command}}
+    <small><a class="pull-right" href="/{{session.request_type|replace("_detail","")}}_bgpmap/{{session.hosts}}/{{session.proto}}?q={{session.request_args}}">View the BGP map</a></small>
+</h3>
 {% if session.request_args != expression|replace("/32","")|replace("/128","") %}
 <i>DNS: <a href="/whois/{{session.request_args}}" class="whois">{{session.request_args}}</a> => <a href="/whois/{{ expression|replace("/32","")|replace("/128","") }}" class="whois">{{expression|replace("/32","")|replace("/128","")}}</a></i><br />
 {% endif %}<br />
@@ -9,6 +12,5 @@
 {{ detail[host]|trim|safe }}
 </pre>
 {% endfor %}
-<a href="/bgpmap/{{bgpmap}}"><img src="/bgpmap/{{bgpmap}}" /></a>
 <br />
 {% endblock %}