From 73809b867dce0a386ecc8a94dd5968a809d27223 Mon Sep 17 00:00:00 2001 From: Phyks Date: Sat, 26 Apr 2014 18:43:25 +0200 Subject: [PATCH] No fail if proxy is not accessible --- README.md | 1 - fetcher.py | 22 ++++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d84f327..1f87950 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,6 @@ A list of ideas and TODO. Don't hesitate to give feedback on the ones you really * if doi does not exist ? * download djvu * test file field for bibtex -* No fail if proxy not accessible * Open * Confirmation for deletion * Rebuild diff --git a/fetcher.py b/fetcher.py index 954aedb..0a22783 100755 --- a/fetcher.py +++ b/fetcher.py @@ -5,9 +5,19 @@ Fetches papers. """ +from __future__ import print_function +import sys import requesocks as requests import params + +def warning(*objs): + """ + Write to stderr + """ + print("WARNING: ", *objs, file=sys.stderr) + + def download_url(url): for proxy in params.proxies: r_proxy = { @@ -15,11 +25,15 @@ def download_url(url): "https": proxy, } - r = requests.get(url, proxies=r_proxy) + try: + r = requests.get(url, proxies=r_proxy) - if r.status_code != 200 or 'pdf' not in r.headers['content-type']: + if r.status_code != 200 or 'pdf' not in r.headers['content-type']: + continue + + return r.content + except: + warning("Proxy "+proxy+" not available.") continue - return r.content - return False