Add multiprocessing on queries

server/ip-ssl-subject-api.py

@@ -17,6 +17,9 @@ import os
 
 import M2Crypto
 
+from tornado.concurrent import run_on_executor
+from concurrent.futures import ThreadPoolExecutor
+
 import tornado.httpserver
 import tornado.ioloop
 import tornado.options
@@ -25,38 +28,46 @@ import tornado.web
 from tornado.options import define, options
 define("port", default=8888, help="run on the given port", type=int)
 certrepo = '/data/certs{}'
-ipmaxsize = 512 #/23
+ipmaxsize = 512  # /23
 servername = 'SSL Certificate API - https://github.com/adulau/crl-monitor'
 
+
 def checksha1(value=False):
     if value is False or len(value) != 40:
         return False
     try:
-	sha1int = int(value, 16)
+        int(value, 16)
     except ValueError:
         return False
 
     return True
 
+
 def bpath(ha=None, level=6):
     if ha is None:
         return False
     fn = ""
-    for i in range(0, level*2, 2):
+    for i in range(0, level * 2, 2):
-        fn = fn + "/"+ ha[i:2+i]
+        fn = fn + "/" + ha[i:2 + i]
     return fn
 
 
 class SSLQueryHandler(tornado.web.RequestHandler):
 
-    def get(self, input):
+    # Default value in Python 3.5
+    # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
+    nb_threads = tornado.process.cpu_count() * 5
+    executor = ThreadPoolExecutor(nb_threads)
+
+    @run_on_executor
+    def run_request(self, q):
         try:
-            #Redis structure Set of (FP) per IP
+            # Redis structure Set of (FP) per IP
             r = redis.StrictRedis(host='127.0.0.1', port=8323)
         except:
-            print "Unable to connect to the Redis server"
+            print("Unable to connect to the Redis server")
             sys.exit(255)
-        subnets = [input]
+        subnets = [q]
         out = {}
         for subnet in subnets:
             if re.findall(r":", subnet):
@@ -97,16 +108,32 @@ class SSLQueryHandler(tornado.web.RequestHandler):
         if not self._finished:
             self.set_header('Content-Type', 'application/json')
             self.set_header('Server', servername)
-            self.write(json.dumps(out))
+            return json.dumps(out)
+
+        def get(self, q):
+            print("Query:", q)
+            try:
+                r = yield self.run_request(q)
+                self.write(r)
+            except Exception as e:
+                print('Something went wrong with {}:\n{}'.format(q, e))
+
 
 class CertificateQueryHandler(tornado.web.RequestHandler):
-    def get(self, input):
+
+    # Default value in Python 3.5
+    # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
+    nb_threads = tornado.process.cpu_count() * 5
+    executor = ThreadPoolExecutor(nb_threads)
+
+    @run_on_executor
+    def run_request(self, q):
         try:
             r = redis.StrictRedis(host='127.0.0.1', port=8323)
         except:
-            print "Unable to connect to the Redis server"
+            print("Unable to connect to the Redis server")
             sys.exit(255)
-	fp = input.lower()
+        fp = q.lower()
         if not checksha1(value=fp):
             self.clear()
             self.set_status(400)
@@ -123,23 +150,33 @@ class CertificateQueryHandler(tornado.web.RequestHandler):
         if not self._finished:
             self.set_header('Content-Type', 'application/json')
             self.set_header('Server', servername)
-            self.write(json.dumps(out))
+            return json.dumps(out)
+
+    def get(self, q):
+        print("Query:", q)
+        try:
+            r = yield self.run_request(q)
+            self.write(r)
+        except Exception as e:
+            print('Something went wrong with {}:\n{}'.format(q, e))
+
 
 class FetchCertificateHandler(tornado.web.RequestHandler):
-    def get(self, input):
-	try:
-	    r = redis.StrictRedis(host='127.0.0.1', port=8323)
-        except:
-            print ("Unable to connect to the Redis server")
-            sys.exit(255)
 
-	#ICSI data
+    # Default value in Python 3.5
+    # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
+    nb_threads = tornado.process.cpu_count() * 5
+    executor = ThreadPoolExecutor(nb_threads)
+
+    @run_on_executor
+    def run_request(self, q):
+        # ICSI data
         try:
             ricsi = redis.StrictRedis(host='localhost', port=6380, db=5)
         except:
-	    print ("Unable to connect to the Redis ICSI notary server")
+            print("Unable to connect to the Redis ICSI notary server")
 
-	fp = input.lower()
+        fp = q.lower()
         if not checksha1(value=fp):
             self.clear()
             self.set_status(400)
@@ -175,9 +212,18 @@ class FetchCertificateHandler(tornado.web.RequestHandler):
         if not self._finished:
             self.set_header('Content-Type', 'application/json')
             self.set_header('Server', servername)
-            self.write(json.dumps(out))
+            return json.dumps(out)
 
-if __name__ == "__main__":
+    def get(self, q):
+        print("Query:", q)
+        try:
+            r = yield self.run_request(q)
+            self.write(r)
+        except Exception as e:
+            print('Something went wrong with {}:\n{}'.format(q, e))
+
+
+def main():
     tornado.options.parse_command_line()
     app = tornado.web.Application(handlers=[
         (r"/query/(.*)", SSLQueryHandler),
@@ -188,3 +234,6 @@ if __name__ == "__main__":
     http_server.listen(options.port)
     tornado.ioloop.IOLoop.instance().start()
 
+
+if __name__ == '__main__':
+    sys.exit(main())