phyks/bmc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

No fail if proxy is not accessible

Browse Source
This commit is contained in:
Phyks 2014-04-26 18:43:25 +02:00
parent 0c8e52844a
commit 73809b867d
2 changed files with 18 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -103,7 +103,6 @@ A list of ideas and TODO. Don't hesitate to give feedback on the ones you really
* if doi does not exist ? * if doi does not exist ?
* download djvu * download djvu
* test file field for bibtex * test file field for bibtex
* No fail if proxy not accessible
* Open * Open
* Confirmation for deletion * Confirmation for deletion
* Rebuild * Rebuild

14
fetcher.py
View File

@ -5,9 +5,19 @@
Fetches papers. Fetches papers.
""" """
from __future__ import print_function
import sys
import requesocks as requests import requesocks as requests
import params import params
def warning(*objs):
"""
Write to stderr
"""
print("WARNING: ", *objs, file=sys.stderr)
def download_url(url): def download_url(url):
for proxy in params.proxies: for proxy in params.proxies:
r_proxy = { r_proxy = {
@ -15,11 +25,15 @@ def download_url(url):
"https": proxy, "https": proxy,
} }
try:
r = requests.get(url, proxies=r_proxy) r = requests.get(url, proxies=r_proxy)
if r.status_code != 200 or 'pdf' not in r.headers['content-type']: if r.status_code != 200 or 'pdf' not in r.headers['content-type']:
continue continue
return r.content return r.content
except:
warning("Proxy "+proxy+" not available.")
continue
return False return False