From a4dfadfe04ad4abbbd93fbab6153381a11f2fb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 2 Oct 2016 22:24:42 +0200 Subject: [PATCH 1/2] Move ip-ssl-subject-api.py --- README.md | 8 ++++---- {bin/x509 => server}/ip-ssl-subject-api.py | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename {bin/x509 => server}/ip-ssl-subject-api.py (100%) diff --git a/README.md b/README.md index f7ca0dd..aa964c6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ crl-monitor =========== -CRL Monitor - X.509 Certificate Revocation List monitoring +CRL Monitor - X.509 Certificate Revocation List monitoring X.509 Subject Cache ================ @@ -17,7 +17,7 @@ If you use the great dumps from [scans.io](https://scans.io/), you can do the fo zcat ./scans-io/data/20141208_certs.gz | python dumpx509subject.py -p 6381 -s ~~~~ -This command parses all the certificates and extract the subjects and imports these into the Redis-compatible database running on TCP port 6381. +This command parses all the certificates and extract the subjects and imports these into the Redis-compatible database running on TCP port 6381. Then you need to import the mapping between scanned IP addresses and the fingerprint of the X.509 certificate seen: @@ -26,14 +26,14 @@ zcat ./scans-io/data/20141208_hosts.gz | python hoststoredis.py -p 6381 -s ~~~~ The above procedure can be repeated with additional scans or you can import multiple scans in parallel using GNU Parallel. - + IP Subnet Lookup in X.509 Subject Cache ================================ ip-ssl-subject.py can query a network subnet and display the known certificate seen and display the X.509 subject if known. ~~~~ -python ./bin/x509/ip-ssl-subject.py -s 199.16.156.0/28 -p 6381 +python ./server/ip-ssl-subject-api.py -s 199.16.156.0/28 -p 6381 ~~~~ ~~~~ diff --git a/bin/x509/ip-ssl-subject-api.py b/server/ip-ssl-subject-api.py similarity index 100% rename from bin/x509/ip-ssl-subject-api.py rename to server/ip-ssl-subject-api.py From 1078d2f159db79c813ea6696bbfb304dcca5a87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 2 Oct 2016 22:25:52 +0200 Subject: [PATCH 2/2] Add multiprocessing on queries --- server/ip-ssl-subject-api.py | 203 ++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 77 deletions(-) diff --git a/server/ip-ssl-subject-api.py b/server/ip-ssl-subject-api.py index 4b2574c..9283da0 100644 --- a/server/ip-ssl-subject-api.py +++ b/server/ip-ssl-subject-api.py @@ -17,6 +17,9 @@ import os import M2Crypto +from tornado.concurrent import run_on_executor +from concurrent.futures import ThreadPoolExecutor + import tornado.httpserver import tornado.ioloop import tornado.options @@ -25,38 +28,46 @@ import tornado.web from tornado.options import define, options define("port", default=8888, help="run on the given port", type=int) certrepo = '/data/certs{}' -ipmaxsize = 512 #/23 +ipmaxsize = 512 # /23 servername = 'SSL Certificate API - https://github.com/adulau/crl-monitor' + def checksha1(value=False): if value is False or len(value) != 40: - return False + return False try: - sha1int = int(value, 16) + int(value, 16) except ValueError: - return False + return False return True + def bpath(ha=None, level=6): if ha is None: return False fn = "" - for i in range(0, level*2, 2): - fn = fn + "/"+ ha[i:2+i] + for i in range(0, level * 2, 2): + fn = fn + "/" + ha[i:2 + i] return fn class SSLQueryHandler(tornado.web.RequestHandler): - def get(self, input): + # Default value in Python 3.5 + # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor + nb_threads = tornado.process.cpu_count() * 5 + executor = ThreadPoolExecutor(nb_threads) + + @run_on_executor + def run_request(self, q): try: - #Redis structure Set of (FP) per IP + # Redis structure Set of (FP) per IP r = redis.StrictRedis(host='127.0.0.1', port=8323) except: - print "Unable to connect to the Redis server" + print("Unable to connect to the Redis server") sys.exit(255) - subnets = [input] + subnets = [q] out = {} for subnet in subnets: if re.findall(r":", subnet): @@ -83,108 +94,146 @@ class SSLQueryHandler(tornado.web.RequestHandler): s = r.smembers(ip) if s: out[str(ip)] = {} - out[str(ip)]['certificates'] = [] - out[str(ip)]['subjects'] = {} + out[str(ip)]['certificates'] = [] + out[str(ip)]['subjects'] = {} for fingerprint in s: subjects = r.smembers(fingerprint) - out[str(ip)]['certificates'].append(fingerprint) + out[str(ip)]['certificates'].append(fingerprint) if subjects: - out[str(ip)]['subjects'][fingerprint] = {} - out[str(ip)]['subjects'][fingerprint]['values'] = [] + out[str(ip)]['subjects'][fingerprint] = {} + out[str(ip)]['subjects'][fingerprint]['values'] = [] for subject in subjects: out[str(ip)]['subjects'][fingerprint]['values'].append(subject) if not self._finished: self.set_header('Content-Type', 'application/json') self.set_header('Server', servername) - self.write(json.dumps(out)) + return json.dumps(out) + + def get(self, q): + print("Query:", q) + try: + r = yield self.run_request(q) + self.write(r) + except Exception as e: + print('Something went wrong with {}:\n{}'.format(q, e)) + class CertificateQueryHandler(tornado.web.RequestHandler): - def get(self, input): + + # Default value in Python 3.5 + # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor + nb_threads = tornado.process.cpu_count() * 5 + executor = ThreadPoolExecutor(nb_threads) + + @run_on_executor + def run_request(self, q): try: r = redis.StrictRedis(host='127.0.0.1', port=8323) except: - print "Unable to connect to the Redis server" + print("Unable to connect to the Redis server") sys.exit(255) - fp = input.lower() - if not checksha1(value=fp): - self.clear() - self.set_status(400) - self.finish('Incorrect format of the certificate fingerprint (expected SHA1 in hex format)') - - out = {} - out['certificate'] = fp - out['seen'] = [] - ips = r.smembers('s:{}'.format(fp)) - out['hits'] = len(ips) - for ip in ips: - out['seen'].append(ip) - - if not self._finished: - self.set_header('Content-Type', 'application/json') - self.set_header('Server', servername) - self.write(json.dumps(out)) - -class FetchCertificateHandler(tornado.web.RequestHandler): - def get(self, input): - try: - r = redis.StrictRedis(host='127.0.0.1', port=8323) - except: - print ("Unable to connect to the Redis server") - sys.exit(255) - - #ICSI data - try: - ricsi = redis.StrictRedis(host='localhost', port=6380, db=5) - except: - print ("Unable to connect to the Redis ICSI notary server") - - fp = input.lower() + fp = q.lower() if not checksha1(value=fp): self.clear() self.set_status(400) self.finish('Incorrect format of the certificate fingerprint (expected SHA1 in hex format)') - certpath = bpath(ha=fp) - certpath = os.path.join(certpath, fp) - certpath = certrepo.format(certpath) - if not os.path.exists(certpath): - self.clear() - self.set_status(400) + out = {} + out['certificate'] = fp + out['seen'] = [] + ips = r.smembers('s:{}'.format(fp)) + out['hits'] = len(ips) + for ip in ips: + out['seen'].append(ip) + + if not self._finished: + self.set_header('Content-Type', 'application/json') + self.set_header('Server', servername) + return json.dumps(out) + + def get(self, q): + print("Query:", q) + try: + r = yield self.run_request(q) + self.write(r) + except Exception as e: + print('Something went wrong with {}:\n{}'.format(q, e)) + + +class FetchCertificateHandler(tornado.web.RequestHandler): + + # Default value in Python 3.5 + # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor + nb_threads = tornado.process.cpu_count() * 5 + executor = ThreadPoolExecutor(nb_threads) + + @run_on_executor + def run_request(self, q): + # ICSI data + try: + ricsi = redis.StrictRedis(host='localhost', port=6380, db=5) + except: + print("Unable to connect to the Redis ICSI notary server") + + fp = q.lower() + if not checksha1(value=fp): + self.clear() + self.set_status(400) + self.finish('Incorrect format of the certificate fingerprint (expected SHA1 in hex format)') + + certpath = bpath(ha=fp) + certpath = os.path.join(certpath, fp) + certpath = certrepo.format(certpath) + if not os.path.exists(certpath): + self.clear() + self.set_status(400) self.finish('Not existing certificate') - cert = M2Crypto.X509.load_cert(certpath, M2Crypto.X509.FORMAT_DER) - out = {} + cert = M2Crypto.X509.load_cert(certpath, M2Crypto.X509.FORMAT_DER) + out = {} out['pem'] = cert.as_pem() out['info'] = {} - out['info']['issuer'] = cert.get_issuer().as_text() + out['info']['issuer'] = cert.get_issuer().as_text() out['info']['subject'] = cert.get_subject().as_text() - out['info']['fingerprint'] = cert.get_fingerprint(md='sha1') - out['info']['keylength'] = cert.get_pubkey().get_rsa().__len__() - out['info']['key'] = cert.get_pubkey().get_rsa().as_pem() - out['info']['not_before'] = cert.get_not_before().get_datetime().isoformat() - out['info']['not_after'] = cert.get_not_after().get_datetime().isoformat() - out['info']['extension'] = {} + out['info']['fingerprint'] = cert.get_fingerprint(md='sha1') + out['info']['keylength'] = cert.get_pubkey().get_rsa().__len__() + out['info']['key'] = cert.get_pubkey().get_rsa().as_pem() + out['info']['not_before'] = cert.get_not_before().get_datetime().isoformat() + out['info']['not_after'] = cert.get_not_after().get_datetime().isoformat() + out['info']['extension'] = {} extcount = cert.get_ext_count() - for i in range(0, extcount): - out['info']['extension'][cert.get_ext_at(i).get_name()] = cert.get_ext_at(i).get_value() - if ricsi.exists(fp): - icsi = ricsi.hgetall(fp) - out['icsi'] = icsi - if not self._finished: - self.set_header('Content-Type', 'application/json') + for i in range(0, extcount): + out['info']['extension'][cert.get_ext_at(i).get_name()] = cert.get_ext_at(i).get_value() + if ricsi.exists(fp): + icsi = ricsi.hgetall(fp) + out['icsi'] = icsi + if not self._finished: + self.set_header('Content-Type', 'application/json') self.set_header('Server', servername) - self.write(json.dumps(out)) + return json.dumps(out) -if __name__ == "__main__": + def get(self, q): + print("Query:", q) + try: + r = yield self.run_request(q) + self.write(r) + except Exception as e: + print('Something went wrong with {}:\n{}'.format(q, e)) + + +def main(): tornado.options.parse_command_line() app = tornado.web.Application(handlers=[ (r"/query/(.*)", SSLQueryHandler), (r"/cquery/(.*)", CertificateQueryHandler), - (r"/cfetch/(.*)", FetchCertificateHandler) + (r"/cfetch/(.*)", FetchCertificateHandler) ]) http_server = tornado.httpserver.HTTPServer(app) http_server.listen(options.port) tornado.ioloop.IOLoop.instance().start() + +if __name__ == '__main__': + sys.exit(main())