phyks/bmc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tabs to spaces

Browse Source
This commit is contained in:
Nathan McCorkle 2013-01-10 21:14:16 -08:00
parent e4074d2b3d
commit 6dcb23e2f8
1 changed files with 70 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
modules/papers.py
View File

@ -39,92 +39,92 @@ def download(phenny, input, verbose=True):
# don't bother if there's nothing there # don't bother if there's nothing there
if len(line) < 5 or (not "http://" in line and not "https://" in line) or not line.startswith("http"): if len(line) < 5 or (not "http://" in line and not "https://" in line) or not line.startswith("http"):
return return
for line in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', line): for line in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', line):
translation_url = "http://localhost:1969/web" translation_url = "http://localhost:1969/web"
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
} }
data = { data = {
"url": line, "url": line,
"sessionid": "what" "sessionid": "what"
} }
data = json.dumps(data) data = json.dumps(data)
response = requests.post(translation_url, data=data, headers=headers) response = requests.post(translation_url, data=data, headers=headers)
if response.status_code == 200: if response.status_code == 200:
# see if there are any attachments # see if there are any attachments
content = json.loads(response.content) content = json.loads(response.content)
item = content[0] item = content[0]
title = item["title"] title = item["title"]
if item.has_key("attachments"): if item.has_key("attachments"):
pdf_url = None pdf_url = None
for attachment in item["attachments"]: for attachment in item["attachments"]:
if attachment.has_key("mimeType") and "application/pdf" in attachment["mimeType"]: if attachment.has_key("mimeType") and "application/pdf" in attachment["mimeType"]:
pdf_url = attachment["url"] pdf_url = attachment["url"]
break break
if pdf_url: if pdf_url:
user_agent = "Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11" user_agent = "Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"
headers = { headers = {
"User-Agent": user_agent, "User-Agent": user_agent,
} }
response = None response = None
if pdf_url.startswith("https://"): if pdf_url.startswith("https://"):
response = requests.get(pdf_url, headers=headers, verify=False) response = requests.get(pdf_url, headers=headers, verify=False)
else: else:
response = requests.get(pdf_url, headers=headers) response = requests.get(pdf_url, headers=headers)
# detect failure # detect failure
if response.status_code == 401: if response.status_code == 401:
phenny.say("HTTP 401 unauthorized " + str(pdf_url)) phenny.say("HTTP 401 unauthorized " + str(pdf_url))
continue continue
elif response.status_code != 200: elif response.status_code != 200:
phenny.say("HTTP " + str(response.status_code) + " " + str(pdf_url)) phenny.say("HTTP " + str(response.status_code) + " " + str(pdf_url))
continue continue
data = response.content data = response.content
# grr.. # grr..
title = title.encode("ascii", "ignore") title = title.encode("ascii", "ignore")
path = os.path.join("/home/bryan/public_html/papers2/paperbot/", title + ".pdf") path = os.path.join("/home/bryan/public_html/papers2/paperbot/", title + ".pdf")
file_handler = open(path, "w") file_handler = open(path, "w")
file_handler.write(data) file_handler.write(data)
file_handler.close() file_handler.close()
filename = requests.utils.quote(title) filename = requests.utils.quote(title)
url = "http://diyhpl.us/~bryan/papers2/paperbot/" + filename + ".pdf" url = "http://diyhpl.us/~bryan/papers2/paperbot/" + filename + ".pdf"
phenny.say(url) phenny.say(url)
continue continue
elif verbose and explicit: elif verbose and explicit:
phenny.say("error: didn't find any pdfs on " + line) phenny.say("error: didn't find any pdfs on " + line)
phenny.say(download_url(line)) phenny.say(download_url(line))
continue continue
elif verbose and explicit: elif verbose and explicit:
phenny.say("error: dunno how to find the pdf on " + line) phenny.say("error: dunno how to find the pdf on " + line)
phenny.say(download_url(line)) phenny.say(download_url(line))
continue continue
elif verbose and explicit: elif verbose and explicit:
if response.status_code == 501: if response.status_code == 501:
if verbose: if verbose:
phenny.say("no translator available, raw dump: " + download_url(line)) phenny.say("no translator available, raw dump: " + download_url(line))
continue continue
else: else:
if verbose: if verbose:
phenny.say("error: HTTP " + str(response.status_code) + " " + download_url(line)) phenny.say("error: HTTP " + str(response.status_code) + " " + download_url(line))
continue continue
else: else:
continue continue
return return
download.commands = ["fetch", "get", "download"] download.commands = ["fetch", "get", "download"]
download.priority = "high" download.priority = "high"
download.rule = r'(.*)' download.rule = r'(.*)'