adulau/DomainClassifier
1
0
Fork 0
mirror of https://github.com/adulau/DomainClassifier.git synced 2024-09-18 10:36:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

python 3.5 upgrade

Browse source
This commit is contained in:
Terrtia 2018-04-18 11:53:22 +02:00
parent 682260259c
commit 907571d502
2 changed files with 21 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
DomainClassifier/domainclassifier.py
View file

@ -8,7 +8,7 @@ import re
import dns.resolver import dns.resolver
import IPy import IPy
import socket import socket
import urllib2 import urllib.request
__author__ = "Alexandre Dulaunoy" __author__ = "Alexandre Dulaunoy"
__copyright__ = "Copyright 2012-2017, Alexandre Dulaunoy" __copyright__ = "Copyright 2012-2017, Alexandre Dulaunoy"
@ -27,7 +27,7 @@ class Extract:
self.presolver = dns.resolver.Resolver() self.presolver = dns.resolver.Resolver()
self.presolver.nameservers = nameservers self.presolver.nameservers = nameservers
self.presolver.lifetime = 1.0 self.presolver.lifetime = 1.0
self.bgprankingserver = 'pdns.circl.lu' self.bgprankingserver = 'pdns.circl.lu' #'bgpranking.circl.lu'
self.vdomain = [] self.vdomain = []
self.listtld = [] self.listtld = []
self.domain = self.potentialdomain() self.domain = self.potentialdomain()
@ -49,7 +49,7 @@ class Extract:
if a: if a:
x = str(a[0]).split("|") x = str(a[0]).split("|")
# why so many spaces? # why so many spaces?
x = map(lambda t: t.replace("\"", "").strip(), x) x = list( map(lambda t: t.replace("\"", "").strip(), x) )
return (x[0], x[2], a[0]) return (x[0], x[2], a[0])
else: else:
return None return None
@ -82,9 +82,9 @@ class Extract:
def __updatelisttld(self): def __updatelisttld(self):
ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
req = urllib2.Request(ianatldlist) req = urllib.request.Request(ianatldlist)
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0') req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0')
tlds = urllib2.urlopen(req).read() tlds = ( urllib.request.urlopen(req).read() ).decode('utf8')
tlds = tlds.split("\n") tlds = tlds.split("\n")
for tld in tlds: for tld in tlds:
self.listtld.append(tld.lower()) self.listtld.append(tld.lower())
@ -274,7 +274,6 @@ class Extract:
for dom in domains: for dom in domains:
if type(dom) == tuple: if type(dom) == tuple:
dom = dom[0] dom = dom[0]
if includefilter.search(dom): if includefilter.search(dom):
self.cleandomain.append(dom) self.cleandomain.append(dom)

32
DomainClassifier/test.py
View file

@ -1,19 +1,19 @@
import domainclassifier import domainclassifier
c = domainclassifier.Extract( rawtext = "www.xxx.com this is a text with a domain called test@foo.lu another test abc.lu something a.b.c.d.e end of 1.2.3.4 foo.be www.belnet.be http://www.cert.be/ www.public.lu www.allo.lu quuxtest www.eurodns.com something-broken-www.google.com www.google.lu trailing test www.facebook.com www.nic.ru www.youporn.com 8.8.8.8 201.1.1.1", nameservers = ['149.13.33.69']) c = domainclassifier.Extract( rawtext = "www.xxx.com this is a text with a domain called test@foo.lu another test abc.lu something a.b.c.d.e end of 1.2.3.4 foo.be www.belnet.be http://www.cert.be/ www.public.lu www.allo.lu quuxtest www.eurodns.com something-broken-www.google.com www.google.lu trailing test www.facebook.com www.nic.ru www.youporn.com 8.8.8.8 201.1.1.1", nameservers = ['8.8.8.8'])
#print c.potentialdomain() #print c.potentialdomain()
print c.validdomain(extended=True) print(c.validdomain(extended=True))
print "US:" print("US:")
print c.localizedomain(cc='US') print(c.localizedomain(cc='US'))
print "LU:" print("LU:")
print c.localizedomain(cc='LU') print(c.localizedomain(cc='LU'))
print "BE:" print("BE:")
print c.localizedomain(cc='BE') print(c.localizedomain(cc='BE'))
print "Ranking:" print("Ranking:")
print c.rankdomain() print(c.rankdomain())
print "List of ip addresses:" print("List of ip addresses:")
print c.ipaddress(extended=True) print(c.ipaddress(extended=True))
print "Include dot.lu:" print("Include dot.lu:")
print c.include(expression=r'\.lu$') print(c.include(expression=r'\.lu$'))
print "Exclude dot.lu:" print("Exclude dot.lu:")
print c.exclude(expression=r'\.lu$') print(c.exclude(expression=r'\.lu$'))