alarig
/
bird-lg
mirror of https://github.com/sileht/bird-lg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge 8aadc7c072 into 6a7bd7f228

This commit is contained in:
dsx 2015-01-27 16:31:41 +00:00
parent 6a7bd7f228 8aadc7c072
commit ec29a34bf7
6 changed files with 300 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
.gitignore vendored
View File

@ -1,2 +1,52 @@
*.pyc
*.pyo
# Python
*.py[co]
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
tmp
develop-eggs
share
.installed.cfg
pip-log.txt
# Nose
.coverage
.tox
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
# Others
*~
*#
.#*
*.orig
data
lg.log*
# Virtualenv
.Python
include
lib
local
man
# Sphinx
doc/build/
# Database
db/*
# Local development
local.yaml
local.ini

10
lg.cfg
View File

@ -30,3 +30,13 @@ AS_NUMBER = {
ASN_ZONE = "asn.cymru.com"
SESSION_KEY = '\xd77\xf9\xfa\xc2\xb5\xcd\x85)`+H\x9d\xeeW\\%\xbe/\xbaT\x89\xe8\xa7'
# Set to True if you're using one BIRD with full view of the network graph
BIRD_HAS_FULL_VIEW = True
# Maximum number of paths to show
MAX_PATHS = 7
# Shorten router names (xxx-yyy-zzz.telecom.local -> xxx-yyy-zzz, provided you specify '.telecom.local' as a value)
ROUTER_NAME_REMOVE = '.telecom.local.'

199
lg.py
View File

@ -20,22 +20,22 @@
#
###
import memcache
import subprocess
import logging
from collections import defaultdict
from logging.handlers import TimedRotatingFileHandler
import re
from urllib2 import urlopen
from urllib import quote, unquote
from urllib2 import urlopen
import json
import logging
import memcache
import random
import re
import subprocess
from toolbox import mask_is_valid, ipv6_is_valid, ipv4_is_valid, resolve, save_cache_pickle, load_cache_pickle, unescape
#from xml.sax.saxutils import escape
from toolbox import mask_is_valid, ipv6_is_valid, ipv4_is_valid, resolve, resolve_ptr, save_cache_pickle, load_cache_pickle, unescape
import pydot
from dns.resolver import NXDOMAIN
from flask import Flask, render_template, jsonify, redirect, session, request, abort, Response, Markup
import pydot
app = Flask(__name__)
app.config.from_pyfile('lg.cfg')
@ -47,16 +47,17 @@ file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
app.logger.addHandler(file_handler)
memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211")
memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
mc = memcache.Client([memcache_server])
def get_asn_from_as(n):
asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com")
try:
data = resolve("AS%s.%s" % (n, asn_zone) ,"TXT").replace("'","").replace('"','')
data = resolve("AS%s.%s" % (n, asn_zone), "TXT").replace("'", "").replace('"', '')
except:
return " "*5
return [ field.strip() for field in data.split("|") ]
return " " * 5
return [field.strip() for field in data.split("|")]
def add_links(text):
@ -69,8 +70,7 @@ def add_links(text):
ret_text = []
for line in text:
# Some heuristic to create link
if line.strip().startswith("BGP.as_path:") or \
line.strip().startswith("Neighbor AS:"):
if line.strip().startswith("BGP.as_path:") or line.strip().startswith("Neighbor AS:"):
ret_text.append(re.sub(r'(\d+)', r'<a href="/whois?q=\1" class="whois">\1</a>', line))
else:
line = re.sub(r'([a-zA-Z0-9\-]*\.([a-zA-Z]{2,3}){1,2})(\s|$)', r'<a href="/whois?q=\1" class="whois">\1</a>\3', line)
@ -111,7 +111,7 @@ def set_session(request_type, hosts, proto, request_args):
def whois_command(query):
server = []
if app.config.get("WHOIS_SERVER", ""):
server = [ "-h", app.config.get("WHOIS_SERVER") ]
server = ["-h", app.config.get("WHOIS_SERVER")]
return subprocess.Popen(['whois'] + server + [query], stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore')
@ -143,7 +143,7 @@ def bird_proxy(host, proto, service, query):
elif not path:
return False, 'Proto "%s" invalid' % proto
else:
url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query))
url = 'http://{}:{}/{}?q={}'.format(app.config['ROUTER_IP'][host][0], port, path, quote(query))
try:
f = urlopen(url)
resultat = f.read()
@ -157,18 +157,18 @@ def bird_proxy(host, proto, service, query):
@app.context_processor
def inject_commands():
commands = [
("traceroute", "traceroute ..."),
("summary", "show protocols"),
("detail", "show protocols ... all"),
("prefix", "show route for ..."),
("prefix_detail", "show route for ... all"),
("prefix_bgpmap", "show route for ... (bgpmap)"),
("where", "show route where net ~ [ ... ]"),
("where_detail", "show route where net ~ [ ... ] all"),
("where_bgpmap", "show route where net ~ [ ... ] (bgpmap)"),
("adv", "show route ..."),
("adv_bgpmap", "show route ... (bgpmap)"),
]
("traceroute", "traceroute ..."),
("summary", "show protocols"),
("detail", "show protocols ... all"),
("prefix", "show route for ..."),
("prefix_detail", "show route for ... all"),
("prefix_bgpmap", "show route for ... (bgpmap)"),
("where", "show route where net ~ [ ... ]"),
("where_detail", "show route where net ~ [ ... ] all"),
("where_bgpmap", "show route where net ~ [ ... ] (bgpmap)"),
("adv", "show route ..."),
("adv_bgpmap", "show route ... (bgpmap)"),
]
commands_dict = {}
for id, text in commands:
commands_dict[id] = text
@ -198,10 +198,12 @@ def incorrect_request(e):
def page_not_found(e):
return render_template('error.html', warnings=["The requested URL was not found on the server."]), 404
def get_query():
q = unquote(request.args.get('q', '').strip())
return q
@app.route("/whois")
def whois():
query = get_query()
@ -317,7 +319,6 @@ def traceroute(hosts, proto):
errors.append("%s" % resultat)
continue
infos[host] = add_links(resultat)
return render_template('traceroute.html', infos=infos, errors=errors)
@ -377,7 +378,7 @@ def get_as_name(_as):
name = mc.get(str('lg_%s' % _as))
if not name:
app.logger.info("asn for as %s not found in memcache", _as)
name = get_asn_from_as(_as)[-1].replace(" ","\r",1)
name = get_asn_from_as(_as)[-1].replace(" ", "\r", 1)
if name:
mc.set(str("lg_%s" % _as), str(name), memcache_expiration)
return "AS%s | %s" % (_as, name)
@ -415,7 +416,7 @@ def show_bgpmap():
def add_node(_as, **kwargs):
if _as not in nodes:
kwargs["label"] = '<<TABLE CELLBORDER="0" BORDER="0" CELLPADDING="0" CELLSPACING="0"><TR><TD ALIGN="CENTER">' + escape(kwargs.get("label", get_as_name(_as))).replace("\r","<BR/>") + "</TD></TR></TABLE>>"
kwargs["label"] = '<<TABLE CELLBORDER="0" BORDER="0" CELLPADDING="0" CELLSPACING="0"><TR><TD ALIGN="CENTER">' + escape(kwargs.get("label", get_as_name(_as))).replace("\r", "<BR/>") + "</TD></TR></TABLE>>"
nodes[_as] = pydot.Node(_as, style="filled", fontsize="10", **kwargs)
graph.add_node(nodes[_as])
return nodes[_as]
@ -433,17 +434,17 @@ def show_bgpmap():
e = edges[edge_tuple]
label_without_star = kwargs["label"].replace("*", "")
labels = e.get_label().split("\r")
labels = e.get_label().split("\r")
if "%s*" % label_without_star not in labels:
labels = [ kwargs["label"] ] + [ l for l in labels if not l.startswith(label_without_star) ]
labels = sorted(labels, cmp=lambda x,y: x.endswith("*") and -1 or 1)
labels = [kwargs["label"]] + [l for l in labels if not l.startswith(label_without_star)]
labels = sorted(labels, cmp=lambda x, y: x.endswith("*") and -1 or 1)
label = escape("\r".join(labels))
e.set_label(label)
return edges[edge_tuple]
for host, asmaps in data.iteritems():
add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
add_node(host, label="%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
as_number = app.config["AS_NUMBER"].get(host, None)
if as_number:
@ -451,8 +452,8 @@ def show_bgpmap():
edge = add_edge(as_number, nodes[host])
edge.set_color("red")
edge.set_style("bold")
#colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93", "#9a0e85", "#56d8e1" ]
# colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93", "#9a0e85", "#56d8e1" ]
previous_as = None
hosts = data.keys()
for host, asmaps in data.iteritems():
@ -468,16 +469,18 @@ def show_bgpmap():
continue
if not hop:
hop = True
if _as not in hosts:
hop_label = _as
if app.config.get('BIRD_HAS_FULL_VIEW', False):
hop = True
hop_label = ''
continue
elif _as not in hosts:
hop_label = _as
if first:
hop_label = hop_label + "*"
continue
else:
hop_label = ""
add_node(_as, fillcolor=(first and "#F5A9A9" or "white"))
if hop_label:
edge = add_edge(nodes[previous_as], nodes[_as], label=hop_label, fontsize="7")
@ -500,7 +503,7 @@ def show_bgpmap():
node = add_node(previous_as)
node.set_shape("box")
#return Response("<pre>" + graph.create_dot() + "</pre>")
# return Response("<pre>" + graph.create_dot() + "</pre>")
return Response(graph.create_png(), mimetype='image/png')
@ -510,10 +513,14 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
path = None
paths = []
net_dest = None
re_via = re.compile(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+')
re_unreachable = re.compile(r'(.*)unreachable\s+\[(\w+)\s+')
for line in text:
line = line.strip()
expr = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+', line)
expr = re_via.search(line)
if expr:
if path:
path.append(net_dest)
@ -533,10 +540,10 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
break
else:
# ugly hack for good printing
path = [ peer_protocol_name ]
# path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))]
expr2 = re.search(r'(.*)unreachable\s+\[(\w+)\s+', line)
path = [peer_protocol_name]
# path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))]
expr2 = re_unreachable.search(line)
if expr2:
if path:
path.append(net_dest)
@ -548,7 +555,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
if line.startswith("BGP.as_path:"):
path.extend(line.replace("BGP.as_path:", "").strip().split(" "))
if path:
path.append(net_dest)
paths.append(path)
@ -556,6 +563,93 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
return paths
def build_as_tree_from_full_view(host, proto, res):
re_chunk_start = re.compile(r'(.*)unreachable\s+\[(.*)\s+.*\s+from\s+(.*)\].*\(.*\)\s\[.*\]')
dest_subnet = None
raw = defaultdict(dict)
for line in res:
line = line.strip()
expr = re_chunk_start.search(line)
if expr:
# Beginning of the BGP reply chunk
if not dest_subnet:
dest_subnet = expr.group(1).strip()
router_tag = expr.group(2).strip()
router_ip = expr.group(3).strip()
try:
router_ip = resolve_ptr(router_ip)
except NXDOMAIN:
# If PTR record can't be found, IP will do too
pass
elif line.startswith('BGP.as_path:'):
# BGP AS path
line = line.replace('BGP.as_path:', '')
line = line.strip()
path = [router_tag, ]
for as_num in line.split(' '):
if as_num:
path.append(as_num)
path_tag = '+'.join(path[1:])
if path_tag not in raw:
raw[path_tag] = list()
raw[path_tag].append(dict(router_tag=router_tag, router_ip=router_ip, path=path))
elif line.startswith('BGP.community:'):
# BGP community
line = line.replace('BGP.community:', '')
line = line.strip()
raw[path_tag][-1]['community'] = line.split(' ')
elif line.startswith('BGP.cluster_list:'):
# BGP cluster size
line = line.replace('BGP.cluster_list:', '')
line = line.strip()
raw[path_tag][-1]['cluster_size'] = len(line.split(' '))
for path_tag in raw:
raw[path_tag] = iter(raw[path_tag])
result = defaultdict(list)
exhausted_tags = set()
existing_paths_num = len(raw)
if len(raw) > app.config.get('MAX_PATHS', 10):
max_paths = existing_paths_num
else:
max_paths = app.config.get('MAX_PATHS', 10)
path_count = 0
while path_count < max_paths:
for path_tag in sorted(raw, key=lambda x: x.count('+')):
if path_tag in exhausted_tags:
continue
try:
path = next(raw[path_tag])
except StopIteration:
exhausted_tags.add(path_tag)
continue
result[path['router_ip']].append(path['path'])
result[path['router_ip']][-1].append(dest_subnet)
path_count += 1
if path_count == max_paths:
break
if path_count == max_paths or len(exhausted_tags) == existing_paths_num:
break
return result
def show_route(request_type, hosts, proto):
expression = get_query()
if not expression:
@ -618,7 +712,10 @@ def show_route(request_type, hosts, proto):
continue
if bgpmap:
detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res)
if app.config['BIRD_HAS_FULL_VIEW']:
detail = build_as_tree_from_full_view(host, proto, res)
else:
detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res)
else:
detail[host] = add_links(res)

3
lgproxy.cfg
View File

@ -5,3 +5,6 @@ LOG_LEVEL="WARNING"
ACCESS_LIST = ["91.224.149.206", "178.33.111.110", "2a01:6600:8081:ce00::1"]
IPV4_SOURCE=""
IPV6_SOURCE=""
FEATURES=['traceroute', 'bird' ]
SOCKET_PATH={4: '/var/run/bird.ctl', 6: '/var/run/bird6.ctl'}

76
lgproxy.py
View File

@ -35,60 +35,69 @@ app = Flask(__name__)
app.debug = app.config["DEBUG"]
app.config.from_pyfile('lgproxy.cfg')
file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight")
file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight")
app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
app.logger.addHandler(file_handler)
@app.before_request
def access_log_before(*args, **kwargs):
app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s"%(k,v) for k,v in request.headers.items()]))
app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s" % (k, v) for k, v in request.headers.items()]))
@app.after_request
def access_log_after(response, *args, **kwargs):
app.logger.info("[%s] reponse %s, %s", request.remote_addr, request.url, response.status_code)
app.logger.info("[%s] reponse %s, %s", request.remote_addr, request.url, response.status_code)
return response
def check_accesslist():
if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
abort(401)
def check_features():
features = app.config.get('FEATURES', [])
if request.endpoint not in features:
abort(401)
@app.route("/traceroute")
@app.route("/traceroute6")
def traceroute():
check_accesslist()
if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'):
traceroute4 = [ 'traceroute' ]
traceroute6 = [ 'traceroute6' ]
else: # For Linux
traceroute4 = [ 'traceroute', '-4' ]
traceroute6 = [ 'traceroute', '-6' ]
traceroute4 = ['traceroute']
traceroute6 = ['traceroute6']
else: # For Linux
traceroute4 = ['traceroute', '-4']
traceroute6 = ['traceroute', '-6']
src = []
if request.path == '/traceroute6':
traceroute = traceroute6
if app.config.get("IPV6_SOURCE",""):
src = [ "-s", app.config.get("IPV6_SOURCE") ]
if request.path == '/traceroute6':
traceroute = traceroute6
if app.config.get("IPV6_SOURCE", ""):
src = ["-s", app.config.get("IPV6_SOURCE")]
else:
traceroute = traceroute4
if app.config.get("IPV4_SOURCE",""):
src = [ "-s", app.config.get("IPV4_SOURCE") ]
else:
traceroute = traceroute4
if app.config.get("IPV4_SOURCE", ""):
src = ["-s", app.config.get("IPV4_SOURCE")]
query = request.args.get("q","")
query = request.args.get("q", "")
query = unquote(query)
if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd'):
options = [ '-a', '-q1', '-w1', '-m15' ]
options = ['-a', '-q1', '-w1', '-m15']
elif sys.platform.startswith('openbsd'):
options = [ '-A', '-q1', '-w1', '-m15' ]
else: # For Linux
options = [ '-A', '-q1', '-N32', '-w1', '-m15' ]
command = traceroute + src + options + [ query ]
result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","<br>")
return result
options = ['-A', '-q1', '-w1', '-m15']
else: # For Linux
options = ['-A', '-q1', '-N32', '-w1', '-m15']
command = traceroute + src + options + [query]
result = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n", "<br>")
return result
@app.route("/bird")
@ -96,18 +105,21 @@ def traceroute():
def bird():
check_accesslist()
if request.path == "/bird": b = BirdSocket(file="/var/run/bird.ctl")
elif request.path == "/bird6": b = BirdSocket(file="/var/run/bird6.ctl")
else: return "No bird socket selected"
if request.path == "/bird":
b = BirdSocket(file=app.config.get('SOCKET_PATH').get(4))
elif request.path == "/bird6":
b = BirdSocket(file=app.config.get('SOCKET_PATH').get(6))
else:
return "No bird socket selected"
query = request.args.get("q","")
query = request.args.get("q", "")
query = unquote(query)
status, result = b.cmd(query)
b.close()
# FIXME: use status
return result
if __name__ == "__main__":
app.logger.info("lgproxy start")

67
toolbox.py
View File

@ -19,26 +19,41 @@
#
###
from dns import resolver
from dns import resolver, reversename
import socket
import pickle
import xml.parsers.expat
from flask import Flask
resolv = resolver.Resolver()
resolv.timeout = 0.5
resolv.lifetime = 1
app = Flask(__name__)
app.config.from_pyfile('lg.cfg')
def resolve(n, q):
return str(resolv.query(n,q)[0])
return str(resolv.query(n, q)[0])
def resolve_ptr(ip):
ptr = str(resolve(reversename.from_address(ip), 'PTR')).lower()
ptr = ptr.replace(app.config.get('ROUTER_NAME_REMOVE', ''), '')
return ptr
def mask_is_valid(n):
if not n:
return True
try:
mask = int(n)
return ( mask >= 1 and mask <= 128)
except:
return False
if not n:
return True
try:
mask = int(n)
return (mask >= 1 and mask <= 128)
except:
return False
def ipv4_is_valid(n):
try:
@ -47,6 +62,7 @@ def ipv4_is_valid(n):
except socket.error:
return False
def ipv6_is_valid(n):
try:
socket.inet_pton(socket.AF_INET6, n)
@ -54,22 +70,25 @@ def ipv6_is_valid(n):
except socket.error:
return False
def save_cache_pickle(filename, data):
output = open(filename, 'wb')
pickle.dump(data, output)
output.close()
def load_cache_pickle(filename, default = None):
try:
pkl_file = open(filename, 'rb')
except IOError:
return default
try:
data = pickle.load(pkl_file)
except:
data = default
pkl_file.close()
return data
def save_cache_pickle(filename, data):
output = open(filename, 'wb')
pickle.dump(data, output)
output.close()
def load_cache_pickle(filename, default=None):
try:
pkl_file = open(filename, 'rb')
except IOError:
return default
try:
data = pickle.load(pkl_file)
except:
data = default
pkl_file.close()
return data
def unescape(s):
want_unicode = False