phyks
/
bmc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix jstor title determination

This commit is contained in:
Bryan Bishop 2013-02-21 17:29:53 -06:00
parent 16c7f4d4db
commit 04644364e2
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
modules/papers.py
View File

@ -218,9 +218,12 @@ def download_url(url):
# not all pages have the <input type="hidden" name="ppv-title"> element
try:
title = tree.xpath("//input[@name='ppv-title']/@value")[0]
title = tree.xpath("//div[@class='hd title']")[0].text
except Exception:
pass
try:
title = tree.xpath("//input[@name='ppv-title']/@value")[0]
except Exception:
pass
# get the document id
document_id = None
@ -234,7 +237,7 @@ def download_url(url):
if document_id.isdigit():
try:
pdf_url = "http://www.jstor.org/stable/pdfplus/" + document_id + ".pdf?acceptTC=true"
new_response = requests.get(pdf_url, header={"User-Agent": "time-machine/1.1"})
new_response = requests.get(pdf_url, headers={"User-Agent": "time-machine/1.1"})
new_content = new_response.content
if "pdf" in new_response.headers["content-type"]:
extension = ".pdf"