Set a default User-Agent as IANA is filtering based on UA of the HTTP request

This commit is contained in:
Alexandre Dulaunoy 2017-07-14 08:16:37 +02:00
parent 6e741ad37c
commit 8738865aa5

View file

@ -82,7 +82,9 @@ class Extract:
def __updatelisttld(self): def __updatelisttld(self):
ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
tlds = urllib2.urlopen(ianatldlist, ianatldlist).read() req = urllib2.Request(ianatldlist)
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0')
tlds = urllib2.urlopen(req).read()
tlds = tlds.split("\n") tlds = tlds.split("\n")
for tld in tlds: for tld in tlds:
self.listtld.append(tld.lower()) self.listtld.append(tld.lower())