Merge branch 'master' of https://github.com/sileht/bird-lg

.gitignore

@@ -1,2 +1,4 @@
 *.pyc
 *.pyo
+lg.cfg
+lgproxy.cfg

README.mkd

@@ -1,6 +1,9 @@
 BIRD-LG
 =======
 
+Overview
+--------
+
 This is a looking glass for the Internet Routing Daemon "Bird".
 
 Software is split in two parts:
@@ -8,7 +11,8 @@ Software is split in two parts:
  - lgproxy.py:
 
    It must be installed and started on all bird nodes. It act as a proxy to make traceroute and bird query on the node.
-   Access restriction to this web service can be done in file "lgproxy.cfg" (only IP address based restriction for now).
+   Access restriction to this web service can be done in file "lgproxy.cfg". Two access restriction methods can be configured:
+   based on source IP address or based on a shared secret. Both methods can be used at the same time.
 
  - lg.py:
 
@@ -33,17 +37,42 @@ Software is split in two parts:
 ```
 
 
-bird-lg depends on :
+Installation
+------------
+
+The web service (lg.py) depends on:
 
  - python-flask  >= 0.8
  - python-dnspython
  - python-pydot
- - python-memcache
  - graphviz
  - whois
- - traceroute
 
-Each services can be embedded in any webserver by following regular python-flask configuration.
+The proxy running on routers (lgproxy.py) depends on:
+
+ - python-flask  >= 0.8
+ - traceroute
+ - ping
+
+Each service can be embedded in any webserver by following regular python-flask configuration.
+It is also possible to run the services directly with python for developping / testing:
+
+    python2 lg.py
+    python2 lgproxy.py
+
+Systemd unit files are provided in the `init/` subdirectory.
+
+
+Configuration
+-------------
+
+On your routers, copy `lgproxy.cfg.example` to `lgproxy.cfg` and edit the values.
+
+On the web host, copy `lg.cfg.example` to `lg.cfg` and edit the values.
+
+
+License
+-------
 
 Source code is under GPL 3.0, powered by Flask, jQuery and Bootstrap.
 
@@ -67,7 +96,8 @@ Happy users
 * https://lg.man-da.de/
 * http://route-server.belwue.net/
 * https://lg.exn.uk/
-* http://lg.meerfarbig.net/
+* https://meerblick.io/
+* https://lg.as49697.net/
 * http://lg.netnation.com/
 * http://lg.edxnetwork.eu/
 * https://lg.hivane.net/
@@ -83,3 +113,4 @@ Happy users
 * https://lg.fullsave.net/
 * http://lg.catnix.net/
 * https://lg.worldstream.nl/
+* https://lg.angolacables.co.ao/

bird.py

@@ -172,6 +172,3 @@ class BirdSocket:
 
 
 __all__ = ['BirdSocketSingleton', 'BirdSocket']
-		
-
-

lg.cfg.example

@@ -1,10 +1,19 @@
+# Configuration file example for lg.py
+# Adapt and copy to lg.cfg
 
+WEBSITE_TITLE="Bird-LG / Looking Glass"
 DEBUG = False
 LOG_FILE="/var/log/lg.log"
 LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0
 
 DOMAIN = "tetaneutral.net"
 
+# Used to optionally restrict access to lgproxy based on a shared secret.
+# Empty string or unset = no shared secret is used to run queries on lgproxies.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
 BIND_IP = "0.0.0.0"
 BIND_PORT = 5000
 
@@ -29,4 +38,5 @@ AS_NUMBER = {
 # DNS zone to query for ASN -> name mapping
 ASN_ZONE = "asn.cymru.com"
 
+# Used for secure session storage, change this
 SESSION_KEY = '\xd77\xf9\xfa\xc2\xb5\xcd\x85)`+H\x9d\xeeW\\%\xbe/\xbaT\x89\xe8\xa7'

lg.py

@@ -22,7 +22,6 @@
 
 import base64
 from datetime import datetime
-import memcache
 import subprocess
 import logging
 from logging.handlers import TimedRotatingFileHandler
@@ -49,13 +48,10 @@ app.config.from_pyfile(args.config_file)
 app.secret_key = app.config["SESSION_KEY"]
 app.debug = app.config["DEBUG"]
 
-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight")
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)
 
-memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211")
-memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
-mc = memcache.Client([memcache_server])
 
 def get_asn_from_as(n):
     asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com")
@@ -149,15 +145,24 @@ def bird_proxy(host, proto, service, query):
         return False, 'Host "%s" invalid' % host
     elif not path:
         return False, 'Proto "%s" invalid' % proto
-    else:
+
-        url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query))
+    url = "http://%s" % (host)
+    if "DOMAIN" in app.config:
+        url = "%s.%s" % (url, app.config["DOMAIN"])
+    url = "%s:%d/%s?" % (url, port, path)
+    if "SHARED_SECRET" in app.config:
+        url = "%ssecret=%s&" % (url, app.config["SHARED_SECRET"])
+    url = "%sq=%s" % (url, quote(query))
+
     try:
         f = urlopen(url)
         resultat = f.read()
         status = True                # retreive remote status
     except IOError:
-            resultat = "Failed retreive url: %s" % url
+        resultat = "Failed to retrieve URL for host %s" % host
+        app.logger.warning("Failed to retrieve URL for host %s: %s", host, url)
         status = False
+
     return status, resultat
 
 
@@ -231,7 +236,7 @@ def whois():
         if m:
             query = query.groupdict()["domain"]
 
-    output = whois_command(query).replace("\n", "<br>")
+    output = whois_command(query)
     return jsonify(output=output, title=query)
 
 
@@ -415,10 +420,7 @@ def show_route_for_bgpmap(hosts, proto):
 
 
 def get_as_name(_as):
-    """return a string that contain the as number following by the as name
+    """Returns a string that contain the as number following by the as name
-
-    It's the use whois database informations
-    # Warning, the server can be blacklisted from ripe is too many requests are done
     """
     if not _as:
         return "AS?????"
@@ -426,12 +428,7 @@ def get_as_name(_as):
     if not _as.isdigit():
         return _as.strip()
 
-    name = mc.get(str('lg_%s' % _as))
-    if not name:
-        app.logger.info("asn for as %s not found in memcache", _as)
     name = get_asn_from_as(_as)[-1].replace(" ", "\r", 1)
-        if name:
-            mc.set(str("lg_%s" % _as), str(name), memcache_expiration)
     return "AS%s | %s" % (_as, name)
 
 
@@ -494,13 +491,15 @@ def show_bgpmap():
             if "%s*" % label_without_star not in labels:
                 labels = [ kwargs["label"] ]  + [ l for l in labels if not l.startswith(label_without_star) ]
                 labels = sorted(labels, cmp=lambda x,y: x.endswith("*") and -1 or 1)
-                
                 label = escape("\r".join(labels))
                 e.set_label(label)
         return edges[edge_tuple]
 
     for host, asmaps in data.iteritems():
+        if "DOMAIN" in app.config:
             add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
+        else:
+            add_node(host, label= "%s" % (host.upper()), shape="box", fillcolor="#F5A9A9")
 
         as_number = app.config["AS_NUMBER"].get(host, None)
         if as_number:
@@ -522,7 +521,13 @@ def show_bgpmap():
             hop_label = ""
             for _as in asmap:
                 if _as == previous_as:
-                    prepend_as[_as] = prepend_as.get(_as, 1) + 1
+                    if not prepend_as.get(_as, None):
+                        prepend_as[_as] = {}
+                    if not prepend_as[_as].get(host, None):
+                        prepend_as[_as][host] = {}
+                    if not prepend_as[_as][host].get(asmap[0], None):
+                        prepend_as[_as][host][asmap[0]] = 1
+                    prepend_as[_as][host][asmap[0]] += 1
                     continue
 
                 if not hop:
@@ -535,8 +540,10 @@ def show_bgpmap():
                     else:
                         hop_label = ""
 
-                
+                if _as == asmap[-1]:
-                add_node(_as, fillcolor=(first and "#F5A9A9" or "white"))
+                    add_node(_as, fillcolor="#F5A9A9", shape="box", )
+                else:
+                    add_node(_as, fillcolor=(first and "#F5A9A9" or "white"), )
                 if hop_label:
                     edge = add_edge(nodes[previous_as], nodes[_as], label=hop_label, fontsize="7")
                 else:
@@ -544,22 +551,19 @@ def show_bgpmap():
 
                 hop_label = ""
 
-                if first:
+                if first or _as == asmap[-1]:
                     edge.set_style("bold")
                     edge.set_color("red")
-                elif edge.get_color() != "red":
+                elif edge.get_style() != "bold":
                     edge.set_style("dashed")
                     edge.set_color(color)
 
                 previous_as = _as
             first = False
 
-    if previous_as:
-        node = add_node(previous_as)
-        node.set_shape("box")
-
     for _as in prepend_as:
-        graph.add_edge(pydot.Edge(*(_as, _as), label=" %dx" % prepend_as[_as], color="grey", fontcolor="grey"))
+        for n in set([ n for h, d in prepend_as[_as].iteritems() for p, n in d.iteritems() ]):
+            graph.add_edge(pydot.Edge(*(_as, _as), label=" %dx" % n, color="grey", fontcolor="grey"))
 
     fmt = request.args.get('fmt', 'png')
     #response = Response("<pre>" + graph.create_dot() + "</pre>")
@@ -583,21 +587,29 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
     path = None
     paths = []
     net_dest = None
+    peer_protocol_name = ""
     for line in text:
         line = line.strip()
 
-        expr = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+', line)
+        expr = re.search(r'(.*)unicast\s+\[(\w+)\s+', line)
         if expr:
+            if expr.group(1).strip():
+                net_dest = expr.group(1).strip()
+            peer_protocol_name = expr.group(2).strip()
+
+        expr2 = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on\s+\S+(\s+\[(\w+)\s+)?', line)
+        if expr2:
             if path:
                 path.append(net_dest)
                 paths.append(path)
                 path = None
 
-            if expr.group(1).strip():
+            if expr2.group(1).strip():
-                net_dest = expr.group(1).strip()
+                net_dest = expr2.group(1).strip()
 
-            peer_ip = expr.group(2).strip()
+            peer_ip = expr2.group(2).strip()
-            peer_protocol_name = expr.group(3).strip()
+            if expr2.group(4):
+                peer_protocol_name = expr2.group(4).strip()
             # Check if via line is a internal route
             for rt_host, rt_ips in app.config["ROUTER_IP"].iteritems():
                 # Special case for internal routing
@@ -609,15 +621,18 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
                 path = [ peer_protocol_name ]
 #                path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))]
 
-        expr2 = re.search(r'(.*)unreachable\s+\[(\w+)\s+', line)
+        expr3 = re.search(r'(.*)unreachable\s+\[(\w+)\s+', line)
-        if expr2:
+        if expr3:
             if path:
                 path.append(net_dest)
                 paths.append(path)
                 path = None
 
-            if expr2.group(1).strip():
+            if path is None:
-                net_dest = expr2.group(1).strip()
+                path = [ expr3.group(2).strip() ]
+
+            if expr3.group(1).strip():
+                net_dest = expr3.group(1).strip()
 
         if line.startswith("BGP.as_path:"):
             ASes = line.replace("BGP.as_path:", "").strip().split(" ")

lgproxy.cfg

@@ -1,12 +0,0 @@
-
-DEBUG=False
-LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
-LOG_LEVEL="WARNING"
-BIND_IP = "0.0.0.0"
-BIND_PORT = 5000
-ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
-IPV4_SOURCE=""
-IPV6_SOURCE=""
-BIRD_SOCKET="/var/run/bird/bird.ctl"
-BIRD6_SOCKET="/var/run/bird/bird6.ctl"
-

lgproxy.cfg.example

@@ -0,0 +1,28 @@
+# Configuration file example for lgproxy.py
+# Adapt and copy to lgproxy.cfg
+
+DEBUG=False
+
+LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
+LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0
+
+BIND_IP = "0.0.0.0"
+BIND_PORT = 5000
+
+# Used to restrict access to lgproxy based on source IP address.
+# Empty list = any IP is allowed to run queries.
+ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
+
+# Used to restrict access to lgproxy based on a shared secret (must also be configured in lg.cfg)
+# Empty string or unset = no shared secret is required to run queries.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
+# Used as source address when running traceroute (optional)
+IPV4_SOURCE="198.51.100.42"
+IPV6_SOURCE="2001:db8:42::1"
+
+BIRD_SOCKET="/var/run/bird/bird.ctl"
+BIRD6_SOCKET="/var/run/bird/bird6.ctl"
+

lgproxy.py

@@ -1,3 +1,4 @@
+#!/usr/bin/python
 # -*- coding: utf-8 -*-
 # vim: ts=4
 ###
@@ -40,7 +41,7 @@ app = Flask(__name__)
 app.debug = app.config["DEBUG"]
 app.config.from_pyfile(args.config_file)
 
-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight") 
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)
 
@@ -53,14 +54,19 @@ def access_log_after(response, *args, **kwargs):
     app.logger.info("[%s] reponse %s, %s", request.remote_addr,  request.url, response.status_code)
     return response
 
-def check_accesslist():
+def check_security():
     if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
+        app.logger.info("Your remote address is not valid")
+        abort(401)
+
+    if app.config.get('SHARED_SECRET') and request.args.get("secret") != app.config["SHARED_SECRET"]:
+        app.logger.info("Your shared secret is not valid")
         abort(401)
 
 @app.route("/traceroute")
 @app.route("/traceroute6")
 def traceroute():
-    check_accesslist()
+    check_security()
 
     if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'):
         traceroute4 = [ 'traceroute' ]
@@ -74,7 +80,6 @@ def traceroute():
         traceroute = traceroute6
         if app.config.get("IPV6_SOURCE", ""):
             src = [ "-s",  app.config.get("IPV6_SOURCE") ]
-
     else: 
         traceroute = traceroute4
         if app.config.get("IPV4_SOURCE",""):
@@ -91,15 +96,13 @@ def traceroute():
         options = [ '-A', '-q1', '-N32', '-w1', '-m15' ]
     command = traceroute + src + options + [ query ]
     result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","<br>")
-    
     return result
 
 
-
 @app.route("/bird")
 @app.route("/bird6")
 def bird():
-    check_accesslist()
+    check_security()
 
     if request.path == "/bird": b = BirdSocket(file=app.config.get("BIRD_SOCKET"))
     elif request.path == "/bird6": b = BirdSocket(file=app.config.get("BIRD6_SOCKET"))

static/js/lg.js

@@ -1,4 +1,4 @@
-
+const noArgReqs = ["summary"];
 
 $(window).unload(function(){
 		$(".progress").show()
@@ -12,7 +12,7 @@ function change_url(loc){
 
 function reload(){
 	loc = "/" + request_type + "/" + hosts + "/" + proto;
-	if (request_type != "summary" ){
+	if (!noArgReqs.includes(request_type)){
 		if( request_args != undefined && request_args != ""){
 			loc = loc + "?q=" + encodeURIComponent(request_args);
 			change_url(loc)
@@ -22,7 +22,7 @@ function reload(){
 	}
 }
 function update_view(){
-	if (request_type == "summary")
+	if (noArgReqs.includes(request_type))
 		$(".navbar-search").hide();
 	else
 		$(".navbar-search").show();
@@ -58,7 +58,7 @@ $(function(){
 			link = $(this).attr('href');
 			$.getJSON(link, function(data) {
 				$(".modal h3").html(data.title);
-			        $(".modal .modal-body > p").html(data.output);
+			        $(".modal .modal-body > p").css("white-space", "pre-line").text(data.output);
 				$(".modal").modal('show');
 			});
 		});

templates/layout.html

@@ -1,7 +1,7 @@
 <!doctype html>
 <html lang="en">
-	<title>{{config.DOMAIN|capitalize}} looking glass</title>
 	<head>
+		<title>{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</title>
 		<meta charset="UTF-8">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap.min.css') }}">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap-responsive.min.css') }}">
@@ -18,7 +18,7 @@
 						<span class="icon-bar"></span>
 						<span class="icon-bar"></span>
 					</a>
-					<a class="brand" href="/">{{config.DOMAIN|capitalize}} / Looking Glass</a>
+					<a class="brand" href="/">{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</a>
 					<div class="navbar nav-collapse">
 						<ul class="nav nav-pills">
 							<li class="navbar-text">Nodes:&nbsp;&nbsp;</li>
@@ -120,7 +120,7 @@
         <script type="text/javascript" src="{{url_for('static', filename='js/DT_bootstrap.js') }}"></script>
 		<script type="text/javascript">
 			request_type = "{{session.request_type}}";
-			request_args = "{{session.request_args|safe}}";
+			request_args = "{{session.request_args}}";
 			hosts = "{{session.hosts}}";
 			proto = "{{session.proto}}";
 			history_query = {{session.history|tojson|safe}};

toolbox.py

@@ -24,9 +24,11 @@ import socket
 import pickle
 import xml.parsers.expat
 
+dns_cache = resolver.LRUCache(max_size=10000)
 resolv = resolver.Resolver()
 resolv.timeout = 0.5
 resolv.lifetime = 1
+resolv.cache = dns_cache
 
 def resolve(n, q):
     return str(resolv.query(n,q)[0])