Change User-Agent if it's default for IA & IS

oduwsdl · Jul 18, 2020 · b4e88b2 · b4e88b2
1 parent 13ecf14
commit b4e88b2
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 11 deletions.
diff --git a/archivenow/__init__.py b/archivenow/__init__.py
@@ -1 +1 @@
-__version__ = '2020.4.1.10.34.36'
+__version__ = '2020.7.18.12.19.44'
diff --git a/archivenow/archivenow.py b/archivenow/archivenow.py
@@ -15,7 +15,7 @@
 
 #from __init__ import __version__ as archiveNowVersion
 
-archiveNowVersion = '2020.4.1.10.34.36'
+archiveNowVersion = '2020.7.18.12.19.44'
 
 # archive handlers path
 PATH = Path(os.path.dirname(os.path.abspath(__file__)))

diff --git a/archivenow/handlers/ia_handler.py b/archivenow/handlers/ia_handler.py
@@ -11,11 +11,13 @@ def push(self, uri_org, p_args=[], session=requests.Session()):
         msg = ''
         try:
             uri = 'https://web.archive.org/save/' + uri_org
-            archiveTodayUserAgent = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}
+            archiveTodayUserAgent = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
+            }
             # push into the archive
             # r = session.get(uri, timeout=120, allow_redirects=True, headers=archiveTodayUserAgent)
 
-            if 'user-agent' in session.headers:
+            if ('user-agent' in session.headers) and (not session.headers['User-Agent'].lower().startswith('python-requests/')):
                 r = session.get(uri, timeout=120, allow_redirects=True)
             else:
                 r = session.get(uri, timeout=120, allow_redirects=True, headers=archiveTodayUserAgent)

diff --git a/archivenow/handlers/is_handler.py b/archivenow/handlers/is_handler.py
@@ -39,9 +39,11 @@ def push(self, uri_org, p_args=[], session=requests.Session()):
 
                 msg = ''
 
-                archiveTodayUserAgent = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)" , "host": host}
-
-                if 'user-agent' in session.headers:
+                archiveTodayUserAgent = {
+                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
+                    "host": host}
+
+                if ('user-agent' in session.headers) and (not session.headers['User-Agent'].lower().startswith('python-requests/')):
                     rid = session.get('https://'+host+'/',timeout=120, allow_redirects=True)
                 else:
                     rid = session.get('https://'+host+'/',timeout=120, allow_redirects=True, headers=archiveTodayUserAgent)
@@ -55,10 +57,17 @@ def push(self, uri_org, p_args=[], session=requests.Session()):
                     msg = "IndexError (" + self.name+ "): unable to extract 'submitid' "
                     raise  
 
-                # push to the archive         
-                r = session.post('https://'+host+'/submit/', timeout=120,
+                # push to the archive
+                if ('user-agent' in session.headers) and (not session.headers['User-Agent'].lower().startswith('python-requests/')):
+                    r = session.post('https://'+host+'/submit/', timeout=120,
                                                                  data={"anyway":"1" , "url":uri_org, "submitid":archiveTodaySubmitId},
                                                                  allow_redirects=True)
+                else:
+                    r = session.post('https://'+host+'/submit/', timeout=120,
+                                                                 data={"anyway":"1" , "url":uri_org, "submitid":archiveTodaySubmitId},
+                                                                 allow_redirects=True,
+                                                                 headers=archiveTodayUserAgent)
+
 
                 r.raise_for_status()
                 # extract the link to the archived copy

diff --git a/archivenow/handlers/mg_handler.py b/archivenow/handlers/mg_handler.py
@@ -4,7 +4,9 @@
 
 
 
-new_header = 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0'
+new_header = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
+
+
 
 class MG_handler(object):
 

diff --git a/archivenow/handlers/st_handler.py b/archivenow/handlers/st_handler.py
@@ -2,7 +2,7 @@
 import requests
 import re
 
-new_header = 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0'
+new_header = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
 
 headers = {'User-Agent': new_header}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = '2020.4.1.10.34.36'
		__version__ = '2020.7.18.12.19.44'
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,7 +4,9 @@



		new_header = 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0'
		new_header = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'



		class MG_handler(object):

Expand Down