Merge branch 'master' into custom-startpage

2024-12-22 12:34:42 +01:00 · 2022-04-05 20:38:43 +02:00 · 2022-04-05 20:38:43 +02:00 · d2653741aa
parent 63a81874a6 f73da84a49
commit d2653741aa
11 changed files with 313 additions and 221 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,4 @@
 *.pyc
 *.pyo
+lg.cfg
+lgproxy.cfg
--- a/README.mkd
+++ b/README.mkd
@ -1,6 +1,9 @@
 BIRD-LG
 =======

+Overview
+--------
+
 This is a looking glass for the Internet Routing Daemon "Bird".

 Software is split in two parts:
@ -8,7 +11,8 @@ Software is split in two parts:
 - lgproxy.py:

   It must be installed and started on all bird nodes. It act as a proxy to make traceroute and bird query on the node.
-   Access restriction to this web service can be done in file "lgproxy.cfg" (only IP address based restriction for now).
+   Access restriction to this web service can be done in file "lgproxy.cfg". Two access restriction methods can be configured:
+   based on source IP address or based on a shared secret. Both methods can be used at the same time.

 - lg.py:

@ -33,17 +37,42 @@ Software is split in two parts:
 ```


-bird-lg depends on :
+Installation
+------------
+
+The web service (lg.py) depends on:

 - python-flask  >= 0.8
 - python-dnspython
 - python-pydot
- - python-memcache
 - graphviz
 - whois
- - traceroute

-Each services can be embedded in any webserver by following regular python-flask configuration.
+The proxy running on routers (lgproxy.py) depends on:
+
+ - python-flask  >= 0.8
+ - traceroute
+ - ping
+
+Each service can be embedded in any webserver by following regular python-flask configuration.
+It is also possible to run the services directly with python for developping / testing:
+
+    python2 lg.py
+    python2 lgproxy.py
+
+Systemd unit files are provided in the `init/` subdirectory.
+
+
+Configuration
+-------------
+
+On your routers, copy `lgproxy.cfg.example` to `lgproxy.cfg` and edit the values.
+
+On the web host, copy `lg.cfg.example` to `lg.cfg` and edit the values.
+
+
+License
+-------

 Source code is under GPL 3.0, powered by Flask, jQuery and Bootstrap.

@ -67,7 +96,8 @@ Happy users
 * https://lg.man-da.de/
 * http://route-server.belwue.net/
 * https://lg.exn.uk/
-* http://lg.meerfarbig.net/
+* https://meerblick.io/
+* https://lg.as49697.net/
 * http://lg.netnation.com/
 * http://lg.edxnetwork.eu/
 * https://lg.hivane.net/
@ -83,4 +113,4 @@ Happy users
 * https://lg.fullsave.net/
 * http://lg.catnix.net/
 * https://lg.worldstream.nl/
-* https://route-server.netshelter.de/
+* https://lg.angolacables.co.ao/
--- a/bird.py
+++ b/bird.py
@ -171,7 +171,4 @@ class BirdSocket:
        return True, parsed_string


-__all__ = ['BirdSocketSingleton' , 'BirdSocket' ]
-		
-
-
+__all__ = ['BirdSocketSingleton', 'BirdSocket']
--- a/lg.cfg.example
+++ b/lg.cfg.example
@ -1,10 +1,19 @@
+# Configuration file example for lg.py
+# Adapt and copy to lg.cfg

+WEBSITE_TITLE="Bird-LG / Looking Glass"
 DEBUG = False
 LOG_FILE="/var/log/lg.log"
 LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0

 DOMAIN = "tetaneutral.net"

+# Used to optionally restrict access to lgproxy based on a shared secret.
+# Empty string or unset = no shared secret is used to run queries on lgproxies.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
 BIND_IP = "0.0.0.0"
 BIND_PORT = 5000

@ -29,6 +38,7 @@ AS_NUMBER = {
 # DNS zone to query for ASN -> name mapping
 ASN_ZONE = "asn.cymru.com"

+# Used for secure session storage, change this
 SESSION_KEY = '\xd77\xf9\xfa\xc2\xb5\xcd\x85)`+H\x9d\xeeW\\%\xbe/\xbaT\x89\xe8\xa7'

 # specifies an alternative start page template for the "/" route.
--- a/lg.py
+++ b/lg.py
@ -22,7 +22,6 @@

 import base64
 from datetime import datetime
-import memcache
 import subprocess
 import logging
 from logging.handlers import TimedRotatingFileHandler
@ -43,18 +42,16 @@ parser = argparse.ArgumentParser()
 parser.add_argument('-c', dest='config_file', help='path to config file', default='lg.cfg')
 args = parser.parse_args()

+
 app = Flask(__name__)
 app.config.from_pyfile(args.config_file)
 app.secret_key = app.config["SESSION_KEY"]
 app.debug = app.config["DEBUG"]

-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight")
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)

-memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211")
-memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
-mc = memcache.Client([memcache_server])

 def get_asn_from_as(n):
    asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com")
@ -148,15 +145,24 @@ def bird_proxy(host, proto, service, query):
        return False, 'Host "%s" invalid' % host
    elif not path:
        return False, 'Proto "%s" invalid' % proto
-    else:
-        url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query))
+
+    url = "http://%s" % (host)
+    if "DOMAIN" in app.config:
+        url = "%s.%s" % (url, app.config["DOMAIN"])
+    url = "%s:%d/%s?" % (url, port, path)
+    if "SHARED_SECRET" in app.config:
+        url = "%ssecret=%s&" % (url, app.config["SHARED_SECRET"])
+    url = "%sq=%s" % (url, quote(query))
+
    try:
        f = urlopen(url)
        resultat = f.read()
        status = True                # retreive remote status
    except IOError:
-            resultat = "Failed retreive url: %s" % url
+        resultat = "Failed to retrieve URL for host %s" % host
+        app.logger.warning("Failed to retrieve URL for host %s: %s", host, url)
        status = False
+
    return status, resultat


@ -175,6 +181,8 @@ def inject_commands():
            ("adv", "show route ..."),
            ("adv_bgpmap", "show route ... (bgpmap)"),
        ]
+
+    commands =  [i for i in commands if i[0] not in app.config.get("BLACKLIST_COMMANDS", [])]
    commands_dict = {}
    for id, text in commands:
        commands_dict[id] = text
@ -242,6 +250,8 @@ SUMMARY_UNWANTED_PROTOS = ["Kernel", "Static", "Device", "Direct"]
@app.route("/summary/<hosts>")
@app.route("/summary/<hosts>/<proto>")
 def summary(hosts, proto="ipv4"):
+    if 'summary' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403

    set_session("summary", hosts, proto, "")
    command = "show protocols"
@ -284,6 +294,9 @@ def summary(hosts, proto="ipv4"):

@app.route("/detail/<hosts>/<proto>")
 def detail(hosts, proto):
+    if 'detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    name = get_query()

    if not name:
@ -313,6 +326,9 @@ def detail(hosts, proto):

@app.route("/traceroute/<hosts>/<proto>")
 def traceroute(hosts, proto):
+    if 'traceroute' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    q = get_query()

    if not q:
@ -346,49 +362,70 @@ def traceroute(hosts, proto):

@app.route("/adv/<hosts>/<proto>")
 def show_route_filter(hosts, proto):
+    if 'adv' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("adv", hosts, proto)


@app.route("/adv_bgpmap/<hosts>/<proto>")
 def show_route_filter_bgpmap(hosts, proto):
+    if 'adv_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("adv_bgpmap", hosts, proto)


@app.route("/where/<hosts>/<proto>")
 def show_route_where(hosts, proto):
+    if 'where' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("where", hosts, proto)


@app.route("/where_detail/<hosts>/<proto>")
 def show_route_where_detail(hosts, proto):
+    if 'where_detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("where_detail", hosts, proto)


@app.route("/where_bgpmap/<hosts>/<proto>")
 def show_route_where_bgpmap(hosts, proto):
+    if 'where_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("where_bgpmap", hosts, proto)


@app.route("/prefix/<hosts>/<proto>")
 def show_route_for(hosts, proto):
+    if 'prefix' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("prefix", hosts, proto)


@app.route("/prefix_detail/<hosts>/<proto>")
 def show_route_for_detail(hosts, proto):
+    if 'prefix_detail' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("prefix_detail", hosts, proto)


@app.route("/prefix_bgpmap/<hosts>/<proto>")
 def show_route_for_bgpmap(hosts, proto):
+    if 'prefix_bgpmap' not in iter(inject_commands()['commands_dict']):
+        return render_template('error.html', errors=["Access denied"]), 403
+
    return show_route("prefix_bgpmap", hosts, proto)


 def get_as_name(_as):
-    """return a string that contain the as number following by the as name
-
-    It's the use whois database informations
-    # Warning, the server can be blacklisted from ripe is too many requests are done
+    """Returns a string that contain the as number following by the as name
    """
    if not _as:
        return "AS?????"
@ -396,12 +433,7 @@ def get_as_name(_as):
    if not _as.isdigit():
        return _as.strip()

-    name = mc.get(str('lg_%s' % _as))
-    if not name:
-        app.logger.info("asn for as %s not found in memcache", _as)
-        name = get_asn_from_as(_as)[-1].replace(" ","\r",1)
-        if name:
-            mc.set(str("lg_%s" % _as), str(name), memcache_expiration)
+    name = get_asn_from_as(_as)[-1].replace(" ", "\r", 1)
    return "AS%s | %s" % (_as, name)


@ -464,13 +496,15 @@ def show_bgpmap():
            if "%s*" % label_without_star not in labels:
                labels = [ kwargs["label"] ]  + [ l for l in labels if not l.startswith(label_without_star) ]
                labels = sorted(labels, cmp=lambda x,y: x.endswith("*") and -1 or 1)
-                
                label = escape("\r".join(labels))
                e.set_label(label)
        return edges[edge_tuple]

    for host, asmaps in data.iteritems():
+        if "DOMAIN" in app.config:
            add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
+        else:
+            add_node(host, label= "%s" % (host.upper()), shape="box", fillcolor="#F5A9A9")

        as_number = app.config["AS_NUMBER"].get(host, None)
        if as_number:
@ -511,7 +545,6 @@ def show_bgpmap():
                    else:
                        hop_label = ""

-                
                if _as == asmap[-1]:
                    add_node(_as, fillcolor="#F5A9A9", shape="box", )
                else:
@ -559,7 +592,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
    path = None
    paths = []
    net_dest = None
-    peer_protocol_name = None
+    peer_protocol_name = ""
    for line in text:
        line = line.strip()

@ -569,7 +602,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
                net_dest = expr.group(1).strip()
            peer_protocol_name = expr.group(2).strip()

-        expr2 = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on\s+\w+(\s+\[(\w+)\s+)?', line)
+        expr2 = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on\s+\S+(\s+\[(\w+)\s+)?', line)
        if expr2:
            if path:
                path.append(net_dest)
--- a/lgproxy.cfg
+++ b/lgproxy.cfg
@ -1,12 +0,0 @@
-
-DEBUG=False
-LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
-LOG_LEVEL="WARNING"
-BIND_IP = "0.0.0.0"
-BIND_PORT = 5000
-ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
-IPV4_SOURCE=""
-IPV6_SOURCE=""
-BIRD_SOCKET="/var/run/bird/bird.ctl"
-BIRD6_SOCKET="/var/run/bird/bird6.ctl"
-
--- a/lgproxy.cfg.example
+++ b/lgproxy.cfg.example
@ -0,0 +1,28 @@
+# Configuration file example for lgproxy.py
+# Adapt and copy to lgproxy.cfg
+
+DEBUG=False
+
+LOG_FILE="/var/log/lg-proxy/lg-proxy.log"
+LOG_LEVEL="WARNING"
+# Keep log history indefinitely by default.
+LOG_NUM_DAYS=0
+
+BIND_IP = "0.0.0.0"
+BIND_PORT = 5000
+
+# Used to restrict access to lgproxy based on source IP address.
+# Empty list = any IP is allowed to run queries.
+ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
+
+# Used to restrict access to lgproxy based on a shared secret (must also be configured in lg.cfg)
+# Empty string or unset = no shared secret is required to run queries.
+SHARED_SECRET="ThisTokenIsNotSecret"
+
+# Used as source address when running traceroute (optional)
+IPV4_SOURCE="198.51.100.42"
+IPV6_SOURCE="2001:db8:42::1"
+
+BIRD_SOCKET="/var/run/bird/bird.ctl"
+BIRD6_SOCKET="/var/run/bird/bird6.ctl"
+
--- a/lgproxy.py
+++ b/lgproxy.py
@ -41,7 +41,7 @@ app = Flask(__name__)
 app.debug = app.config["DEBUG"]
 app.config.from_pyfile(args.config_file)

-file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight") 
+file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight", backupCount=app.config.get("LOG_NUM_DAYS", 0))
 app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
 app.logger.addHandler(file_handler)

@ -54,14 +54,19 @@ def access_log_after(response, *args, **kwargs):
    app.logger.info("[%s] reponse %s, %s", request.remote_addr,  request.url, response.status_code)
    return response

-def check_accesslist():
+def check_security():
    if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
+        app.logger.info("Your remote address is not valid")
+        abort(401)
+
+    if app.config.get('SHARED_SECRET') and request.args.get("secret") != app.config["SHARED_SECRET"]:
+        app.logger.info("Your shared secret is not valid")
        abort(401)

@app.route("/traceroute")
@app.route("/traceroute6")
 def traceroute():
-    check_accesslist()
+    check_security()

    if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'):
        traceroute4 = [ 'traceroute' ]
@ -73,9 +78,8 @@ def traceroute():
    src = []
    if request.path == '/traceroute6':
        traceroute = traceroute6
-	if app.config.get("IPV6_SOURCE",""):
+        if app.config.get("IPV6_SOURCE", ""):
            src = [ "-s",  app.config.get("IPV6_SOURCE") ]
-
    else: 
        traceroute = traceroute4
        if app.config.get("IPV4_SOURCE",""):
@ -92,15 +96,13 @@ def traceroute():
        options = [ '-A', '-q1', '-N32', '-w1', '-m15' ]
    command = traceroute + src + options + [ query ]
    result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","<br>")
-    
    return result


-
@app.route("/bird")
@app.route("/bird6")
 def bird():
-    check_accesslist()
+    check_security()

    if request.path == "/bird": b = BirdSocket(file=app.config.get("BIRD_SOCKET"))
    elif request.path == "/bird6": b = BirdSocket(file=app.config.get("BIRD6_SOCKET"))
--- a/static/js/lg.js
+++ b/static/js/lg.js
@ -1,4 +1,4 @@
-
+const noArgReqs = ["summary"];

 $(window).unload(function(){
 		$(".progress").show()
@ -12,7 +12,7 @@ function change_url(loc){

 function reload(){
 	loc = "/" + request_type + "/" + hosts + "/" + proto;
-	if (request_type != "summary" ){
+	if (!noArgReqs.includes(request_type)){
 		if( request_args != undefined && request_args != ""){
 			loc = loc + "?q=" + encodeURIComponent(request_args);
 			change_url(loc)
@ -22,7 +22,7 @@ function reload(){
 	}
 }
 function update_view(){
-	if (request_type == "summary")
+	if (noArgReqs.includes(request_type))
 		$(".navbar-search").hide();
 	else
 		$(".navbar-search").show();
--- a/templates/layout.html
+++ b/templates/layout.html
@ -1,7 +1,7 @@
 <!doctype html>
 <html lang="en">
-	<title>{{config.DOMAIN|capitalize}} looking glass</title>
 	<head>
+		<title>{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</title>
 		<meta charset="UTF-8">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap.min.css') }}">
 		<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/bootstrap-responsive.min.css') }}">
@ -18,7 +18,7 @@
 						<span class="icon-bar"></span>
 						<span class="icon-bar"></span>
 					</a>
-					<a class="brand" href="/">{{config.DOMAIN|capitalize}} / Looking Glass</a>
+					<a class="brand" href="/">{{config.WEBSITE_TITLE|default("Bird-LG / Looking Glass") }}</a>
 					<div class="navbar nav-collapse">
 						<ul class="nav nav-pills">
 							<li class="navbar-text">Nodes:&nbsp;&nbsp;</li>
@ -120,7 +120,7 @@
        <script type="text/javascript" src="{{url_for('static', filename='js/DT_bootstrap.js') }}"></script>
 		<script type="text/javascript">
 			request_type = "{{session.request_type}}";
-			request_args = "{{session.request_args|safe}}";
+			request_args = "{{session.request_args}}";
 			hosts = "{{session.hosts}}";
 			proto = "{{session.proto}}";
 			history_query = {{session.history|tojson|safe}};
--- a/toolbox.py
+++ b/toolbox.py
@ -24,9 +24,11 @@ import socket
 import pickle
 import xml.parsers.expat

+dns_cache = resolver.LRUCache(max_size=10000)
 resolv = resolver.Resolver()
 resolv.timeout = 0.5
 resolv.lifetime = 1
+resolv.cache = dns_cache

 def resolve(n, q):
    return str(resolv.query(n,q)[0])