Merge branch 'master' into custom-startpage

.gitignore

@@ -1,2 +1,4 @@
 *.pyc
 *.pyo
+lg.cfg
+lgproxy.cfg

README.mkd

@@ -1,6 +1,9 @@
 BIRD-LG
 =======
 
+Overview
+--------
+
 This is a looking glass for the Internet Routing Daemon "Bird".
 
 Software is split in two parts:
@@ -8,7 +11,8 @@ Software is split in two parts:
  - lgproxy.py:
 
    It must be installed and started on all bird nodes. It act as a proxy to make traceroute and bird query on the node.
-   Access restriction to this web service can be done in file "lgproxy.cfg" (only IP address based restriction for now).
+   Access restriction to this web service can be done in file "lgproxy.cfg". Two access restriction methods can be configured:
+   based on source IP address or based on a shared secret. Both methods can be used at the same time.
 
  - lg.py:
 
@@ -33,17 +37,42 @@ Software is split in two parts:
 ```
 
 
-bird-lg depends on :
+Installation
+------------
+
+The web service (lg.py) depends on:
 
  - python-flask  >= 0.8
  - python-dnspython
  - python-pydot
- - python-memcache
  - graphviz
  - whois
- - traceroute
 
-Each services can be embedded in any webserver by following regular python-flask configuration.
+The proxy running on routers (lgproxy.py) depends on:
+
+ - python-flask  >= 0.8
+ - traceroute
+ - ping
+
+Each service can be embedded in any webserver by following regular python-flask configuration.
+It is also possible to run the services directly with python for developping / testing:
+
+    python2 lg.py
+    python2 lgproxy.py
+
+Systemd unit files are provided in the `init/` subdirectory.
+
+
+Configuration
+-------------
+
+On your routers, copy `lgproxy.cfg.example` to `lgproxy.cfg` and edit the values.
+
+On the web host, copy `lg.cfg.example` to `lg.cfg` and edit the values.
+
+
+License
+-------
 
 Source code is under GPL 3.0, powered by Flask, jQuery and Bootstrap.
 
@@ -67,7 +96,8 @@ Happy users
 * https://lg.man-da.de/
 * http://route-server.belwue.net/
 * https://lg.exn.uk/
-* http://lg.meerfarbig.net/
+* https://meerblick.io/
+* https://lg.as49697.net/
 * http://lg.netnation.com/
 * http://lg.edxnetwork.eu/
 * https://lg.hivane.net/
@@ -83,4 +113,4 @@ Happy users
 * https://lg.fullsave.net/
 * http://lg.catnix.net/
 * https://lg.worldstream.nl/
-* https://route-server.netshelter.de/
+* https://lg.angolacables.co.ao/

bird.py

@@ -172,6 +172,3 @@ class BirdSocket:
 
 
 __all__ = ['BirdSocketSingleton', 'BirdSocket']
-		
-
-

lg.cfg.example

@@ -1,10 +1,19 @@
+# Configuration file example for lg.py
+# Adapt and copy to lg.cfg
 
+WEBSITE_TITLE="Bird-LG / Looking Glass"
 DEBUG = False
 LOG_FILE="/var/log/lg.log"
 LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0
 
 DOMAIN = "tetaneutral.net"
 
+# Used to optionally restrict access to lgproxy based on a shared secret.
+# Empty string or unset = no shared secret is used to run queries on lgproxies.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
 BIND_IP = "0.0.0.0"
 BIND_PORT = 5000
 
@@ -29,6 +38,7 @@ AS_NUMBER = {
 # DNS zone to query for ASN -> name mapping
 ASN_ZONE = "asn.cymru.com"
 
+# Used for secure session storage, change this
 SESSION_KEY = '\xd77\xf9\xfa\xc2\xb5\xcd\x85)`+H\x9d\xeeW\\%\xbe/\xbaT\x89\xe8\xa7'
 
 # specifies an alternative start page template for the "/" route.

lg.py

@@ -22,7 +22,6 @@
 
 import base64
 from datetime import datetime
-import memcache
 import subprocess
 import logging
 from logging.handlers import TimedRotatingFileHandler
@@ -43,18 +42,16 @@ parser = argparse.ArgumentParser()
 parser.add_argument('-c', dest='config_file', help='path to config file', default='lg.cfg')
 args = parser.parse_args()
 
+
 app = Flask(__name__)
 app.config.from_pyfile(args.config_file)
 app.secret_key = app.config["SESSION_KEY"]
 app.debug = app.config["DEBUG"]
 
-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight")
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)
 
-memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211")
-memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
-mc = memcache.Client([memcache_server])
 
 def get_asn_from_as(n):
     asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com")
@@ -148,15 +145,24 @@ def bird_proxy(host, proto, service, query):
         return False, 'Host "%s" invalid' % host
     elif not path:
         return False, 'Proto "%s" invalid' % proto
-    else:
+
-        url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query))
+    url = "http://%s" % (host)
+    if "DOMAIN" in app.config:
+        url = "%s.%s" % (url, app.config["DOMAIN"])
+    url = "%s:%d/%s?" % (url, port, path)
+    if "SHARED_SECRET" in app.config:
+        url = "%ssecret=%s&" % (url, app.config["SHARED_SECRET"])
+    url = "%sq=%s" % (url, quote(query))
+
     try:
         f = urlopen(url)
         resultat = f.read()
         status = True                # retreive remote status
     except IOError:
-            resultat = "Failed retreive url: %s" % url
+        resultat = "Failed to retrieve URL for host %s" % host
+        app.logger.warning("Failed to retrieve URL for host %s: %s", host, url)
         status = False
+
     return status, resultat
 
 
@@ -175,6 +181,8 @@ def inject_commands():
             ("adv", "show route ..."),
             ("adv_bgpmap", "show route ... (bgpmap)"),
         ]
+
+    commands =  [i for i in commands if i[0] not in app.config.get("BLACKLIST_COMMANDS", [])]
     commands_dict = {}
     for id, text in commands:
         commands_dict[id] = text
@@ -242,6 +250,8 @@ SUMMARY_UNWANTED_PROTOS = ["Kernel", "Static", "Device", "Direct"]
 @app.route("/summary/<hosts>")
 @app.route("/summary/<hosts>/<proto>")
 def summary(hosts, proto="ipv4"):
+    if 'summary' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
 
     set_session("summary", hosts, proto, "")
     command = "show protocols"
@@ -284,6 +294,9 @@ def summary(hosts, proto="ipv4"):
 
 @app.route("/detail/<hosts>/<proto>")
 def detail(hosts, proto):
+    if 'detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     name = get_query()
 
     if not name:
@@ -313,6 +326,9 @@ def detail(hosts, proto):
 
 @app.route("/traceroute/<hosts>/<proto>")
 def traceroute(hosts, proto):
+    if 'traceroute' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     q = get_query()
 
     if not q:
@@ -346,49 +362,70 @@ def traceroute(hosts, proto):
 
 @app.route("/adv/<hosts>/<proto>")
 def show_route_filter(hosts, proto):
+    if 'adv' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("adv", hosts, proto)
 
 
 @app.route("/adv_bgpmap/<hosts>/<proto>")
 def show_route_filter_bgpmap(hosts, proto):
+    if 'adv_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("adv_bgpmap", hosts, proto)
 
 
 @app.route("/where/<hosts>/<proto>")
 def show_route_where(hosts, proto):
+    if 'where' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("where", hosts, proto)
 
 
 @app.route("/where_detail/<hosts>/<proto>")
 def show_route_where_detail(hosts, proto):
+    if 'where_detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("where_detail", hosts, proto)
 
 
 @app.route("/where_bgpmap/<hosts>/<proto>")
 def show_route_where_bgpmap(hosts, proto):
+    if 'where_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("where_bgpmap", hosts, proto)
 
 
 @app.route("/prefix/<hosts>/<proto>")
 def show_route_for(hosts, proto):
+    if 'prefix' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("prefix", hosts, proto)
 
 
 @app.route("/prefix_detail/<hosts>/<proto>")
 def show_route_for_detail(hosts, proto):
+    if 'prefix_detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("prefix_detail", hosts, proto)
 
 
 @app.route("/prefix_bgpmap/<hosts>/<proto>")
 def show_route_for_bgpmap(hosts, proto):
+    if 'prefix_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
     return show_route("prefix_bgpmap", hosts, proto)
 
 
 def get_as_name(_as):
-    """return a string that contain the as number following by the as name
+    """Returns a string that contain the as number following by the as name
-
-    It's the use whois database informations
-    # Warning, the server can be blacklisted from ripe is too many requests are done
     """
     if not _as:
         return "AS?????"
@@ -396,12 +433,7 @@ def get_as_name(_as):
     if not _as.isdigit():
         return _as.strip()
 
-    name = mc.get(str('lg_%s' % _as))
-    if not name:
-        app.logger.info("asn for as %s not found in memcache", _as)
     name = get_asn_from_as(_as)[-1].replace(" ", "\r", 1)
-        if name:
-            mc.set(str("lg_%s" % _as), str(name), memcache_expiration)
     return "AS%s | %s" % (_as, name)
 
 
@@ -464,13 +496,15 @@ def show_bgpmap():
             if "%s*" % label_without_star not in labels:
                 labels = [ kwargs["label"] ]  + [ l for l in labels if not l.startswith(label_without_star) ]
                 labels = sorted(labels, cmp=lambda x,y: x.endswith("*") and -1 or 1)
-                
                 label = escape("\r".join(labels))
                 e.set_label(label)
         return edges[edge_tuple]
 
     for host, asmaps in data.iteritems():
+        if "DOMAIN" in app.config:
             add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
+        else:
+            add_node(host, label= "%s" % (host.upper()), shape="box", fillcolor="#F5A9A9")
 
         as_number = app.config["AS_NUMBER"].get(host, None)
         if as_number:
@@ -511,7 +545,6 @@ def show_bgpmap():
                     else:
                         hop_label = ""
 
-                
                 if _as == asmap[-1]:
                     add_node(_as, fillcolor="#F5A9A9", shape="box", )
                 else:
@@ -559,7 +592,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
     path = None
     paths = []
     net_dest = None
-    peer_protocol_name = None
+    peer_protocol_name = ""
     for line in text:
         line = line.strip()
 
@@ -569,7 +602,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
                 net_dest = expr.group(1).strip()
             peer_protocol_name = expr.group(2).strip()
 
-        expr2 = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on\s+\w+(\s+\[(\w+)\s+)?', line)
+        expr2 = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on\s+\S+(\s+\[(\w+)\s+)?', line)
         if expr2:
             if path:
                 path.append(net_dest)

lgproxy.cfg

@@ -1,12 +0,0 @@
-
-DEBUG=False
-LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
-LOG_LEVEL="WARNING"
-BIND_IP = "0.0.0.0"
-BIND_PORT = 5000
-ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
-IPV4_SOURCE=""
-IPV6_SOURCE=""
-BIRD_SOCKET="/var/run/bird/bird.ctl"
-BIRD6_SOCKET="/var/run/bird/bird6.ctl"
-

lgproxy.cfg.example

@@ -0,0 +1,28 @@
+# Configuration file example for lgproxy.py
+# Adapt and copy to lgproxy.cfg
+
+DEBUG=False
+
+LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
+LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0
+
+BIND_IP = "0.0.0.0"
+BIND_PORT = 5000
+
+# Used to restrict access to lgproxy based on source IP address.
+# Empty list = any IP is allowed to run queries.
+ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
+
+# Used to restrict access to lgproxy based on a shared secret (must also be configured in lg.cfg)
+# Empty string or unset = no shared secret is required to run queries.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
+# Used as source address when running traceroute (optional)
+IPV4_SOURCE="198.51.100.42"
+IPV6_SOURCE="2001:db8:42::1"
+
+BIRD_SOCKET="/var/run/bird/bird.ctl"
+BIRD6_SOCKET="/var/run/bird/bird6.ctl"
+

lgproxy.py

@@ -41,7 +41,7 @@ app = Flask(__name__)
 app.debug = app.config["DEBUG"]
 app.config.from_pyfile(args.config_file)
 
-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight") 
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)
 
@@ -54,14 +54,19 @@ def access_log_after(response, *args, **kwargs):
     app.logger.info("[%s] reponse %s, %s", request.remote_addr,  request.url, response.status_code)
     return response
 
-def check_accesslist():
+def check_security():
     if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
+        app.logger.info("Your remote address is not valid")
+        abort(401)
+
+    if app.config.get('SHARED_SECRET') and request.args.get("secret") != app.config["SHARED_SECRET"]:
+        app.logger.info("Your shared secret is not valid")
         abort(401)
 
 @app.route("/traceroute")
 @app.route("/traceroute6")
 def traceroute():
-    check_accesslist()
+    check_security()
 
     if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'):
         traceroute4 = [ 'traceroute' ]
@@ -75,7 +80,6 @@ def traceroute():
         traceroute = traceroute6
         if app.config.get("IPV6_SOURCE", ""):
             src = [ "-s",  app.config.get("IPV6_SOURCE") ]
-
     else: 
         traceroute = traceroute4
         if app.config.get("IPV4_SOURCE",""):
@@ -92,15 +96,13 @@ def traceroute():
         options = [ '-A', '-q1', '-N32', '-w1', '-m15' ]
     command = traceroute + src + options + [ query ]
     result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","<br>")
-    
     return result
 
 
-
 @app.route("/bird")
 @app.route("/bird6")
 def bird():
-    check_accesslist()
+    check_security()
 
     if request.path == "/bird": b = BirdSocket(file=app.config.get("BIRD_SOCKET"))
     elif request.path == "/bird6": b = BirdSocket(file=app.config.get("BIRD6_SOCKET"))

static/js/lg.js

@@ -1,4 +1,4 @@
-
+const noArgReqs = ["summary"];
 
 $(window).unload(function(){
 		$(".progress").show()
@@ -12,7 +12,7 @@ function change_url(loc){
 
 function reload(){
 	loc = "/" + request_type + "/" + hosts + "/" + proto;
-	if (request_type != "summary" ){
+	if (!noArgReqs.includes(request_type)){
 		if( request_args != undefined && request_args != ""){
 			loc = loc + "?q=" + encodeURIComponent(request_args);
 			change_url(loc)
@@ -22,7 +22,7 @@ function reload(){
 	}
 }
 function update_view(){
-	if (request_type == "summary")
+	if (noArgReqs.includes(request_type))
 		$(".navbar-search").hide();
 	else
 		$(".navbar-search").show();

templates/layout.html

@@ -1,7 +1,7 @@
 <!doctype html>
 <html lang="en">
-	<title>{{config.DOMAIN|capitalize}} looking glass</title>
 	<head>
+		<title>{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</title>
 		<meta charset="UTF-8">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap.min.css') }}">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap-responsive.min.css') }}">
@@ -18,7 +18,7 @@
 						<span class="icon-bar"></span>
 						<span class="icon-bar"></span>
 					</a>
-					<a class="brand" href="/">{{config.DOMAIN|capitalize}} / Looking Glass</a>
+					<a class="brand" href="/">{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</a>
 					<div class="navbar nav-collapse">
 						<ul class="nav nav-pills">
 							<li class="navbar-text">Nodes:&nbsp;&nbsp;</li>
@@ -120,7 +120,7 @@
         <script type="text/javascript" src="{{url_for('static', filename='js/DT_bootstrap.js') }}"></script>
 		<script type="text/javascript">
 			request_type = "{{session.request_type}}";
-			request_args = "{{session.request_args|safe}}";
+			request_args = "{{session.request_args}}";
 			hosts = "{{session.hosts}}";
 			proto = "{{session.proto}}";
 			history_query = {{session.history|tojson|safe}};

toolbox.py

@@ -24,9 +24,11 @@ import socket
 import pickle
 import xml.parsers.expat
 
+dns_cache = resolver.LRUCache(max_size=10000)
 resolv = resolver.Resolver()
 resolv.timeout = 0.5
 resolv.lifetime = 1
+resolv.cache = dns_cache
 
 def resolve(n, q):
     return str(resolv.query(n,q)[0])