adulau/DomainClassifier
1
0
Fork 0
mirror of https://github.com/adulau/DomainClassifier.git synced 2024-11-07 11:56:25 +00:00
Code Issues Projects Releases Packages Wiki Activity

Set a default User-Agent as IANA is filtering based on UA of the HTTP request

Browse source
This commit is contained in:
Alexandre Dulaunoy 2017-07-14 08:16:37 +02:00
parent 6e741ad37c
commit 8738865aa5
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
DomainClassifier/domainclassifier.py
View file

@ -82,7 +82,9 @@ class Extract:
def __updatelisttld(self): def __updatelisttld(self):
ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
tlds = urllib2.urlopen(ianatldlist, ianatldlist).read() req = urllib2.Request(ianatldlist)
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0')
tlds = urllib2.urlopen(req).read()
tlds = tlds.split("\n") tlds = tlds.split("\n")
for tld in tlds: for tld in tlds:
self.listtld.append(tld.lower()) self.listtld.append(tld.lower())