133 lines
4.3 KiB
Python
133 lines
4.3 KiB
Python
"""
|
|
This file contains functions to download locally some papers, eventually using
|
|
a proxy.
|
|
"""
|
|
import socket
|
|
import sys
|
|
import urllib
|
|
|
|
import socks
|
|
|
|
|
|
# Default socket to use, if no proxy is used
|
|
DEFAULT_SOCKET = socket.socket
|
|
|
|
|
|
def _download_helper(url):
|
|
"""
|
|
Handle the download of an URL, using the proxy currently set in \
|
|
:mod:`socks`.
|
|
|
|
:param url: The URL to download.
|
|
:returns: A tuple of the raw content of the downloaded data and its \
|
|
associated content-type. Returns None if it was \
|
|
unable to download the document.
|
|
"""
|
|
# Try to fetch the URL using the current proxy
|
|
try:
|
|
request = urllib.request.urlopen(url)
|
|
try:
|
|
size = int(dict(request.info())['content-length'].strip())
|
|
except KeyError:
|
|
try:
|
|
size = int(dict(request.info())['Content-Length'].strip())
|
|
except KeyError:
|
|
size = 0
|
|
# Download the document
|
|
doc = b""
|
|
doc_size = 0
|
|
while True:
|
|
buf = request.read(1024)
|
|
if buf:
|
|
doc += buf
|
|
doc_size += len(buf)
|
|
if size != 0:
|
|
# Write progress bar on stdout
|
|
done = int(50 * doc_size / size)
|
|
sys.stdout.write("\r[%s%s]" %
|
|
('='*done, ' '*(50-done)))
|
|
sys.stdout.write(" "+str(int(float(done)/52*100))+"%")
|
|
sys.stdout.flush()
|
|
else:
|
|
break
|
|
# Fetch content type
|
|
contenttype = None
|
|
contenttype_req = None
|
|
try:
|
|
contenttype_req = dict(request.info())['content-type']
|
|
except KeyError:
|
|
try:
|
|
contenttype_req = dict(request.info())['Content-Type']
|
|
except KeyError:
|
|
return None
|
|
if 'pdf' in contenttype_req:
|
|
contenttype = 'pdf'
|
|
elif 'djvu' in contenttype_req:
|
|
contenttype = 'djvu'
|
|
|
|
# Check content type and status code are ok
|
|
if request.getcode() != 200 or contenttype is None:
|
|
# Else, try with the next available proxy
|
|
return None
|
|
|
|
# Return a tuple of the downloaded content and the content-type
|
|
return (doc, contenttype)
|
|
# If an exception occurred, continue with next available proxy
|
|
except (urllib.error.URLError, socket.error, ValueError):
|
|
return None
|
|
|
|
|
|
def download(url, proxies=None):
|
|
"""
|
|
Download a PDF or DJVU document from a url, eventually using proxies.
|
|
|
|
:params url: The URL to the PDF/DJVU document to fetch.
|
|
:params proxies: An optional list of proxies to use. Proxies will be \
|
|
used sequentially. Proxies should be a list of proxy strings. \
|
|
Do not forget to include ``""`` (empty string) in the list if \
|
|
you want to try direct fetching without any proxy.
|
|
|
|
:returns: A tuple of the raw content of the downloaded data and its \
|
|
associated content-type. Returns ``(None, None)`` if it was \
|
|
unable to download the document.
|
|
|
|
>>> download("http://arxiv.org/pdf/1312.4006.pdf") # doctest: +SKIP
|
|
"""
|
|
# Handle default argument
|
|
if proxies is None:
|
|
proxies = [""]
|
|
|
|
# Loop over all available connections
|
|
for proxy in proxies:
|
|
# Handle no proxy case
|
|
if proxy == "":
|
|
socket.socket = DEFAULT_SOCKET
|
|
# Handle SOCKS proxy
|
|
elif proxy.startswith('socks'):
|
|
if proxy[5] == '4':
|
|
proxy_type = socks.SOCKS4
|
|
else:
|
|
proxy_type = socks.SOCKS5
|
|
proxy = proxy[proxy.find('://') + 3:]
|
|
try:
|
|
proxy, port = proxy.split(':')
|
|
except ValueError:
|
|
port = None
|
|
socks.set_default_proxy(proxy_type, proxy, port)
|
|
socket.socket = socks.socksocket
|
|
# Handle generic HTTP proxy
|
|
else:
|
|
try:
|
|
proxy, port = proxy.split(':')
|
|
except ValueError:
|
|
port = None
|
|
socks.set_default_proxy(socks.HTTP, proxy, port)
|
|
socket.socket = socks.socksocket
|
|
|
|
downloaded = _download_helper(url)
|
|
if downloaded is not None:
|
|
return downloaded
|
|
|
|
# In case of running out of proxies, return (None, None)
|
|
return (None, None)
|