From 8738865aa576d3be92fae73038e84081106d634a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 14 Jul 2017 08:16:37 +0200 Subject: [PATCH] Set a default User-Agent as IANA is filtering based on UA of the HTTP request --- DomainClassifier/domainclassifier.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DomainClassifier/domainclassifier.py b/DomainClassifier/domainclassifier.py index 7e56728..7c20045 100644 --- a/DomainClassifier/domainclassifier.py +++ b/DomainClassifier/domainclassifier.py @@ -82,7 +82,9 @@ class Extract: def __updatelisttld(self): ianatldlist = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" - tlds = urllib2.urlopen(ianatldlist, ianatldlist).read() + req = urllib2.Request(ianatldlist) + req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0') + tlds = urllib2.urlopen(req).read() tlds = tlds.split("\n") for tld in tlds: self.listtld.append(tld.lower())