From 5a8ea0750fea1917b2b25846b98cfa0d53c2f0f0 Mon Sep 17 00:00:00 2001 From: Phyks Date: Mon, 26 May 2014 16:12:21 +0200 Subject: [PATCH] License + Consolidating fetcher.py --- README.md | 3 ++- arxiv2bib.py | 0 backend.py | 13 +++++++++++-- bmc.py | 19 ++++++++++++------- fetcher.py | 30 ++++++++++++++++++++++-------- params.py.example | 1 + search.py | 7 ++++++- tools.py | 10 ++++++++++ 8 files changed, 64 insertions(+), 19 deletions(-) mode change 100755 => 100644 arxiv2bib.py diff --git a/README.md b/README.md index 59de9ba..ed6c322 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The resync option will check that all bibtex entries have a corresponding file a ## License -All the source code I wrote is under a `no-alcoohol beer-ware license`. All functions that I didn't write myself are under the original license and their origin is specified in the function itself. +All the source code I wrote is under a `no-alcohol beer-ware license`. All functions that I didn't write myself are under the original license and their origin is specified in the function itself. ``` * -------------------------------------------------------------------------------- * "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): @@ -138,6 +138,7 @@ Here are some sources of inspirations for this project : A list of ideas and TODO. Don't hesitate to give feedback on the ones you really want or to propose your owns. +60. Unittest 70. Export of bibtex 80. Search engine 85. Anti-duplicate ? diff --git a/arxiv2bib.py b/arxiv2bib.py old mode 100755 new mode 100644 diff --git a/backend.py b/backend.py index abfe178..da2b466 100644 --- a/backend.py +++ b/backend.py @@ -1,5 +1,14 @@ -#!/usr/bin/env python2 -# coding=utf8 +# -*- coding: utf8 -*- +# ----------------------------------------------------------------------------- +# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): +# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice +# you can do whatever you want with this stuff (and you can also do whatever +# you want with this stuff without retaining it, but that's not cool...). If we +# meet some day, and you think this stuff is worth it, you can buy me a +# beer soda in return. +# Phyks +# ----------------------------------------------------------------------------- + import os import re diff --git a/bmc.py b/bmc.py index 407afef..db081c7 100755 --- a/bmc.py +++ b/bmc.py @@ -87,14 +87,19 @@ def addFile(src, filetype, manual, autoconfirm, tag): isbn = False if not manual: - if filetype == 'article' or filetype is None: - doi = fetcher.findDOI(src) - if doi is False and (filetype == 'article' or filetype is None): - arxiv = fetcher.findArXivId(src) + try: + if filetype == 'article' or filetype is None: + doi = fetcher.findDOI(src) + if doi is False and (filetype == 'article' or filetype is None): + arxiv = fetcher.findArXivId(src) - if filetype == 'book' or (doi is False and arxiv is False and - filetype is None): - isbn = fetcher.findISBN(src) + if filetype == 'book' or (doi is False and arxiv is False and + filetype is None): + isbn = fetcher.findISBN(src) + except KeyboardInterrupt: + doi = False + arxiv = False + isbn = False if doi is False and isbn is False and arxiv is False: if filetype is None: diff --git a/fetcher.py b/fetcher.py index ed17916..402e5b4 100644 --- a/fetcher.py +++ b/fetcher.py @@ -1,5 +1,14 @@ -#!/usr/bin/env python2 -# coding=utf8 +# -*- coding: utf8 -*- +# ----------------------------------------------------------------------------- +# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): +# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice +# you can do whatever you want with this stuff (and you can also do whatever +# you want with this stuff without retaining it, but that's not cool...). If we +# meet some day, and you think this stuff is worth it, you can buy me a +# beer soda in return. +# Phyks +# ----------------------------------------------------------------------------- + import isbntools import re @@ -55,10 +64,10 @@ def download(url): return False -isbn_re = re.compile(r"isbn (([0-9]{3}[ -])?[0-9][ -][0-9]{2}[ -][0-9]{6}[ -][0-9])", +#isbn_re = re.compile(r"isbn (([0-9]{3}[ -])?[0-9][ -][0-9]{2}[ -][0-9]{6}[ -][0-9])", +isbn_re = re.compile(r'isbn ((?:[0-9]{3}[ -]?)?[0-9]{1,5}[ -]?[0-9]{1,7}[ -]?[0-9]{1,6}[- ]?[0-9])', re.IGNORECASE) - def findISBN(src): """Search for a valid ISBN in src. @@ -100,7 +109,10 @@ def findISBN(src): def isbn2Bib(isbn): """Tries to get bibtex entry from an ISBN number""" # Default merges results from worldcat.org and google books - return fmtbib('bibtex', isbntools.meta(isbn, 'default')) + try: + return fmtbib('bibtex', isbntools.meta(isbn, 'default')) + except: + return '' doi_re = re.compile('(?<=doi)/?:?\s?[0-9\.]{7}/\S*[0-9]', re.IGNORECASE) @@ -129,8 +141,9 @@ def findDOI(src): else: return False + extractfull = '' while totext.poll() is None: - extractfull = totext.stdout.readline() + extractfull += totext.stdout.readline().strip() extractDOI = doi_re.search(extractfull.lower().replace('Œ', '-')) if not extractDOI: # PNAS fix @@ -214,8 +227,9 @@ def findArXivId(src): else: return False + extractfull = '' while totext.poll() is None: - extractfull = totext.stdout.readline() + extractfull += totext.stdout.readline().strip() extractID = arXiv_re.search(extractfull) if extractID: totext.terminate() @@ -250,7 +264,7 @@ def arXiv2Bib(arxiv): except: pass return tools.parsed2Bibtex(fetched_bibtex) - return False + return '' HAL_re = re.compile(r'(hal-\d{8}), version (\d+)') diff --git a/params.py.example b/params.py.example index 25fdd1f..21bbf57 100644 --- a/params.py.example +++ b/params.py.example @@ -1,3 +1,4 @@ +# -*- coding: utf8 -*- # The folder in which the papers should be stored # /!\ Keep the trailing slash /!\ folder = "/home/phyks/Papers/" diff --git a/search.py b/search.py index 01a06f0..d6da1a6 100644 --- a/search.py +++ b/search.py @@ -1,3 +1,4 @@ +# -*- coding: utf8 -*- """Search query parser Modified by Phyks, 2014-05-18. Original source code is here: @@ -122,7 +123,11 @@ class SearchQueryParser: operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr ).setResultsName("or") | operatorAnd) - return operatorOr.parseString + operatorQ = Forward() + operatorQ << Group(operatorOr + Suppress('=') + + operatorOr).setResultsName('field') + + return operatorQ.parseString def evaluateAnd(self, argument): return self.evaluate(argument[0]).intersection(self.evaluate(argument[1])) diff --git a/tools.py b/tools.py index 59cd931..0d9e868 100644 --- a/tools.py +++ b/tools.py @@ -1,4 +1,14 @@ # -*- coding: utf8 -*- +# ----------------------------------------------------------------------------- +# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): +# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice +# you can do whatever you want with this stuff (and you can also do whatever +# you want with this stuff without retaining it, but that's not cool...). If we +# meet some day, and you think this stuff is worth it, you can buy me a +# beer soda in return. +# Phyks +# ----------------------------------------------------------------------------- + from __future__ import print_function import os