mirror of
https://github.com/adulau/DomainClassifier.git
synced 2024-11-07 11:56:25 +00:00
Set a default User-Agent as IANA is filtering based on UA of the HTTP request
This commit is contained in:
parent
6e741ad37c
commit
8738865aa5
1 changed files with 3 additions and 1 deletions
|
@ -82,7 +82,9 @@ class Extract:
|
|||
|
||||
def __updatelisttld(self):
|
||||
ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
|
||||
tlds = urllib2.urlopen(ianatldlist, ianatldlist).read()
|
||||
req = urllib2.Request(ianatldlist)
|
||||
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0')
|
||||
tlds = urllib2.urlopen(req).read()
|
||||
tlds = tlds.split("\n")
|
||||
for tld in tlds:
|
||||
self.listtld.append(tld.lower())
|
||||
|
|
Loading…
Reference in a new issue