No fail if proxy is not accessible
This commit is contained in:
parent
0c8e52844a
commit
73809b867d
@ -103,7 +103,6 @@ A list of ideas and TODO. Don't hesitate to give feedback on the ones you really
|
||||
* if doi does not exist ?
|
||||
* download djvu
|
||||
* test file field for bibtex
|
||||
* No fail if proxy not accessible
|
||||
* Open
|
||||
* Confirmation for deletion
|
||||
* Rebuild
|
||||
|
22
fetcher.py
22
fetcher.py
@ -5,9 +5,19 @@
|
||||
Fetches papers.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import requesocks as requests
|
||||
import params
|
||||
|
||||
|
||||
def warning(*objs):
|
||||
"""
|
||||
Write to stderr
|
||||
"""
|
||||
print("WARNING: ", *objs, file=sys.stderr)
|
||||
|
||||
|
||||
def download_url(url):
|
||||
for proxy in params.proxies:
|
||||
r_proxy = {
|
||||
@ -15,11 +25,15 @@ def download_url(url):
|
||||
"https": proxy,
|
||||
}
|
||||
|
||||
r = requests.get(url, proxies=r_proxy)
|
||||
try:
|
||||
r = requests.get(url, proxies=r_proxy)
|
||||
|
||||
if r.status_code != 200 or 'pdf' not in r.headers['content-type']:
|
||||
if r.status_code != 200 or 'pdf' not in r.headers['content-type']:
|
||||
continue
|
||||
|
||||
return r.content
|
||||
except:
|
||||
warning("Proxy "+proxy+" not available.")
|
||||
continue
|
||||
|
||||
return r.content
|
||||
|
||||
return False
|
||||
|
Loading…
Reference in New Issue
Block a user