new: [multiple MMDB file support] MMDB server can now load a list of

MMDB files + new GeoOpen file including ASN details added as example

GeoOpen files for country only -> https://cra.circl.lu/opendata/geo-open/mmdb-country/
GeoOpen files with country and ASN/ASN description -> https://cra.circl.lu/opendata/geo-open/mmdb-country-asn/
This commit is contained in:
Alexandre Dulaunoy 2022-02-06 11:33:17 +01:00
parent d885f6bc3b
commit 9ce8603f33
Signed by: adulau
GPG key ID: 09E2CD4944E6CBCD
2 changed files with 40 additions and 27 deletions

View file

@ -15,7 +15,7 @@ from wsgiref.simple_server import make_server
import falcon import falcon
import maxminddb import maxminddb
version = "0.1" version = "0.5"
config = configparser.ConfigParser() config = configparser.ConfigParser()
config.read('../etc/server.conf') config.read('../etc/server.conf')
mmdb_file = config['global'].get('mmdb_file') mmdb_file = config['global'].get('mmdb_file')
@ -23,6 +23,8 @@ pubsub = config['global'].getboolean('lookup_pubsub')
port = config['global'].getint('port') port = config['global'].getint('port')
country_file = config['global'].get('country_file') country_file = config['global'].get('country_file')
mmdb_files = mmdb_file.split(",")
with open(country_file) as j: with open(country_file) as j:
country_info = json.load(j) country_info = json.load(j)
@ -30,16 +32,19 @@ if pubsub:
import redis import redis
rdb = redis.Redis(host='127.0.0.1') rdb = redis.Redis(host='127.0.0.1')
meta = {} mmdbs = []
q = maxminddb.open_database(mmdb_file, maxminddb.MODE_MEMORY) for mmdb_file in mmdb_files:
meta['description'] = q.metadata().description meta = {}
meta['build_db'] = time.strftime( meta['reader'] = maxminddb.open_database(mmdb_file, maxminddb.MODE_MEMORY)
'%Y-%m-%d %H:%M:%S', time.localtime(q.metadata().build_epoch) meta['description'] = meta['reader'].metadata().description
) meta['build_db'] = time.strftime(
meta['db_source'] = q.metadata().database_type '%Y-%m-%d %H:%M:%S', time.localtime(meta['reader'].metadata().build_epoch)
meta['nb_nodes'] = q.metadata().node_count )
meta['db_source'] = meta['reader'].metadata().database_type
meta['nb_nodes'] = meta['reader'].metadata().node_count
mmdbs.append(meta)
print(mmdbs)
def validIPAddress(IP: str) -> bool: def validIPAddress(IP: str) -> bool:
try: try:
type(ip_address(IP)) type(ip_address(IP))
@ -72,14 +77,18 @@ class GeoLookup:
resp.media = "IPv4 or IPv6 address is in an incorrect format. Dotted decimal for IPv4 or textual representation for IPv6 are required." resp.media = "IPv4 or IPv6 address is in an incorrect format. Dotted decimal for IPv4 or textual representation for IPv6 are required."
return return
pubLookup(value=f'{value} via {ips} using {ua}') pubLookup(value=f'{value} via {ips} using {ua}')
georesult = q.get(value) for mmdb in mmdbs:
ret.append(georesult) m = {}
georesult['meta'] = meta georesult = mmdb['reader'].get(value)
georesult['ip'] = value m = mmdb.copy()
if georesult['country']['iso_code'] != 'None': del m['reader']
georesult['country_info'] = countryLookup(country=georesult['country']['iso_code']) georesult['meta'] = m
else: georesult['ip'] = value
georesult['country_info'] = {} if georesult['country']['iso_code'] != 'None':
georesult['country_info'] = countryLookup(country=georesult['country']['iso_code'])
else:
georesult['country_info'] = {}
ret.append(georesult)
resp.media = ret resp.media = ret
return return
@ -88,14 +97,18 @@ class MyGeoLookup:
def on_get(self, req, resp): def on_get(self, req, resp):
ret = [] ret = []
ips = req.access_route ips = req.access_route
georesult = q.get(ips[0]) for mmdb in mmdbs:
ret.append(georesult) m = {}
georesult['meta'] = meta georesult = mmdb['reader'].get(ips[0])
georesult['ip'] = ips[0] m = mmdb.copy()
if georesult['country']['iso_code'] != 'None': del m['reader']
georesult['country_info'] = countryLookup(country=georesult['country']['iso_code']) georesult['meta'] = m
else: georesult['ip'] = ips[0]
georesult['country_info'] = {} if georesult['country']['iso_code'] != 'None':
georesult['country_info'] = countryLookup(country=georesult['country']['iso_code'])
else:
georesult['country_info'] = {}
ret.append(georesult)
resp.media = ret resp.media = ret
return return

View file

@ -1,5 +1,5 @@
[global] [global]
mmdb_file = /home/adulau/git/mmdb-server/db/GeoOpen-country.mmd mmdb_file = /home/adulau/git/mmdb-server/db/GeoOpen-Country.mmdb,/home/adulau/git/mmdb-server/db/GeoOpen-Country-ASN.mmdb
country_file = /home/adulau/git/mmdb-server/db/country.json country_file = /home/adulau/git/mmdb-server/db/country.json
lookup_pubsub = no lookup_pubsub = no
port = 8000 port = 8000