diff --git a/LICENSE b/LICENSE index a4123d0..d5508b4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,21 @@ -* -------------------------------------------------------------------------------- -* "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -* Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice you -* can do whatever you want with this stuff (and you can also do whatever you want -* with this stuff without retaining it, but that's not cool...). If we meet some -* day, and you think this stuff is worth it, you can buy me a beer soda -* in return. -* Phyks -* --------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Phyks (Lucas Verney) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/commands.py b/backend/commands.py new file mode 100644 index 0000000..e88b897 --- /dev/null +++ b/backend/commands.py @@ -0,0 +1,59 @@ +import tempfile + +from backend import tools +from libbmc import fetcher + + +def download(url, filetype, manual, autoconfirm, tag): + """ + Download a given URL and add it to the library. + + :param url: URL to download. + :param filetype: paper / book / ``None``. + :param manual: Whether BibTeX should be fetched automatically. + :param autoconfirm: Whether import should be made silent or not. + :param tag: A tag for this file. + :returns: The name of the downloaded file once imported, \ + or ``None`` in case of error. + """ + # Download the paper + print("Downloading %s…" % (url,)) + dl, contenttype = fetcher.download(url) + + if dl is not None: + print("Download finished.") + + # Store it to a temporary file + try: + tmp = tempfile.NamedTemporaryFile(suffix='.%s' % (contenttype,)) + with open(tmp.name, 'wb+') as fh: + fh.write(dl) + + # And add it as a normal paper from now on + new_name = import_file(tmp.name, filetype, manual, + autoconfirm, tag) + if new_name is None: + return None + else: + return new_name + finally: + tmp.close() + else: + tools.warning("Could not fetch %s." % (url,)) + return None + + +def import_file(src, filetype, manual, autoconfirm, tag, rename=True): + """ + Add a file to the library. + + :param src: The path of the file to import. + :param filetype: paper / book / ``None``. + :param manual: Whether BibTeX should be fetched automatically. + :param autoconfirm: Whether import should be made silent or not. + :param tag: A tag for this file. + :param rename: TODO + :returns: The name of the imported file, or ``None`` in case of error. + """ + # TODO + pass diff --git a/libbmc/config.py b/backend/config.py similarity index 96% rename from libbmc/config.py rename to backend/config.py index 1360639..3c78f6c 100644 --- a/libbmc/config.py +++ b/backend/config.py @@ -1,12 +1,3 @@ -from __future__ import unicode_literals -import os -import errno -import imp -import inspect -import json -import sys -import libbmc.tools as tools - # List of available options (in ~/.config/bmc/bmc.json file): # * folder : folder in which papers are stored # * proxies : list of proxies to use, e.g. ['', "socks5://localhost:4711"] @@ -23,9 +14,17 @@ import libbmc.tools as tools # %v = arXiv version (e.g. '-v1') or nothing if not an arXiv paper # You can add your custom masks to rename files by adding functions in -# ~/.config/masks.py. +# ~/.config/bmc/masks.py. # E.g. : def f(x): x.replace('test', 'some_expr') +import os +import errno +import imp +import inspect +import json +import sys + +from backend import tools def make_sure_path_exists(path): try: @@ -86,7 +85,7 @@ class Config(): folder_exists = make_sure_path_exists(self.get("folder")) except OSError: tools.warning("Unable to create paper storage folder.") - sys.exit(1) + raise self.load_masks() def save(self): @@ -98,7 +97,7 @@ class Config(): separators=(',', ': '))) except IOError: tools.warning("Could not write config file.") - sys.exit(1) + raise def load_masks(self): if os.path.isfile(self.config_path + "masks.py"): diff --git a/backend/tools.py b/backend/tools.py new file mode 100644 index 0000000..6cb7ad6 --- /dev/null +++ b/backend/tools.py @@ -0,0 +1,8 @@ +import sys + + +def warning(*objs): + """ + Write warnings to stderr. + """ + print("WARNING: ", *objs, file=sys.stderr) diff --git a/bmc.py b/bmc.py index ded6b05..e3836b6 100755 --- a/bmc.py +++ b/bmc.py @@ -1,507 +1,74 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -from __future__ import unicode_literals +#!/usr/bin/env python3 import argparse import os -import shutil -import subprocess import sys -import tempfile -import bibtexparser -from codecs import open -from libbmc.config import Config -from libbmc import backend -from libbmc import fetcher -from libbmc import tearpages -from libbmc import tools + +from backend import commands +from backend import tools +from backend.config import Config + +# TODO: Handle config -config = Config() -EDITOR = os.environ.get('EDITOR') if os.environ.get('EDITOR') else 'vim' +# Load EDITOR variable +EDITOR = os.environ.get("EDITOR") -def checkBibtex(filename, bibtex_string): - print("The bibtex entry found for "+filename+" is:") - - bibtex = bibtexparser.loads(bibtex_string) - bibtex = bibtex.entries_dict - try: - bibtex = bibtex[list(bibtex.keys())[0]] - # Check entries are correct - if "title" not in bibtex: - raise AssertionError - if "authors" not in bibtex and "author" not in bibtex: - raise AssertionError - if "year" not in bibtex: - raise AssertionError - # Print the bibtex and confirm - print(tools.parsed2Bibtex(bibtex)) - check = tools.rawInput("Is it correct? [Y/n] ") - except KeyboardInterrupt: - sys.exit() - except (IndexError, KeyError, AssertionError): - print("Missing author, year or title in bibtex.") - check = 'n' - - try: - old_filename = bibtex['file'] - except KeyError: - old_filename = False - - while check.lower() == 'n': - with tempfile.NamedTemporaryFile(suffix=".tmp") as tmpfile: - tmpfile.write(bibtex_string.encode('utf-8')) - tmpfile.flush() - subprocess.call([EDITOR, tmpfile.name]) - tmpfile.seek(0) - bibtex = bibtexparser.loads(tmpfile.read().decode('utf-8')+"\n") - - bibtex = bibtex.entries_dict - try: - bibtex = bibtex[list(bibtex.keys())[0]] - except (IndexError, KeyError): - tools.warning("Invalid bibtex entry") - bibtex_string = '' - tools.rawInput("Press Enter to go back to editor.") - continue - if('authors' not in bibtex and 'title' not in bibtex and 'year' not in - bibtex): - tools.warning("Invalid bibtex entry") - bibtex_string = '' - tools.rawInput("Press Enter to go back to editor.") - continue - - if old_filename is not False and 'file' not in bibtex: - tools.warning("Invalid bibtex entry. No filename given.") - tools.rawInput("Press Enter to go back to editor.") - check = 'n' - else: - bibtex_string = tools.parsed2Bibtex(bibtex) - print("\nThe bibtex entry for "+filename+" is:") - print(bibtex_string) - check = tools.rawInput("Is it correct? [Y/n] ") - if old_filename is not False and old_filename != bibtex['file']: - try: - print("Moving file to new location…") - shutil.move(old_filename, bibtex['file']) - except shutil.Error: - tools.warning("Unable to move file "+old_filename+" to " + - bibtex['file']+". You should check it manually.") - - return bibtex - - -def addFile(src, filetype, manual, autoconfirm, tag, rename=True): +def parse_args(): """ - Add a file to the library + Build a parser and parse arguments of command line. + + :returns: Parsed arguments from the parser. """ - doi = False - arxiv = False - isbn = False - - if not manual: - try: - if filetype == 'article' or filetype is None: - id_type, article_id = fetcher.findArticleID(src) - if id_type == "DOI": - doi = article_id - elif id_type == "arXiv": - arxiv = article_id - - if filetype == 'book' or (doi is False and arxiv is False and - filetype is None): - isbn = fetcher.findISBN(src) - except KeyboardInterrupt: - doi = False - arxiv = False - isbn = False - - if doi is False and isbn is False and arxiv is False: - if filetype is None: - tools.warning("Could not determine the DOI nor the arXiv id nor " + - "the ISBN for "+src+". Switching to manual entry.") - doi_arxiv_isbn = '' - while(doi_arxiv_isbn not in - ['doi', 'arxiv', 'isbn', 'manual', 'skip']): - doi_arxiv_isbn = (tools.rawInput("DOI / arXiv " + - "/ ISBN / manual / skip? "). - lower()) - if doi_arxiv_isbn == 'doi': - doi = tools.rawInput('DOI? ') - elif doi_arxiv_isbn == 'arxiv': - arxiv = tools.rawInput('arXiv id? ') - elif doi_arxiv_isbn == 'isbn': - isbn = tools.rawInput('ISBN? ') - elif doi_arxiv_isbn == 'skip': - return False - elif filetype == 'article': - tools.warning("Could not determine the DOI nor the arXiv id for " + - src+", switching to manual entry.") - doi_arxiv = '' - while doi_arxiv not in ['doi', 'arxiv', 'manual', 'skip']: - doi_arxiv = (tools.rawInput("DOI / arXiv / manual / skip? "). - lower()) - if doi_arxiv == 'doi': - doi = tools.rawInput('DOI? ') - elif doi_arxiv == 'arxiv': - arxiv = tools.rawInput('arXiv id? ') - elif doi_arxiv == 'skip': - return False - elif filetype == 'book': - isbn_manual = '' - while isbn_manual not in ['isbn', 'manual', 'skip']: - isbn_manual = tools.rawInput("ISBN / manual / skip? ").lower() - if isbn_manual == 'isbn': - isbn = (tools.rawInput('ISBN? '). - replace(' ', ''). - replace('-', '')) - elif isbn_manual == 'skip': - return False - elif doi is not False: - print("DOI for "+src+" is "+doi+".") - elif arxiv is not False: - print("ArXiv id for "+src+" is "+arxiv+".") - elif isbn is not False: - print("ISBN for "+src+" is "+isbn+".") - - if doi is not False and doi != '': - # Add extra \n for bibtexparser - bibtex = fetcher.doi2Bib(doi).strip().replace(',', ",\n")+"\n" - elif arxiv is not False and arxiv != '': - bibtex = fetcher.arXiv2Bib(arxiv).strip().replace(',', ",\n")+"\n" - elif isbn is not False and isbn != '': - # Idem - bibtex = fetcher.isbn2Bib(isbn).strip()+"\n" - else: - bibtex = '' - - bibtex = bibtexparser.loads(bibtex) - bibtex = bibtex.entries_dict - if len(bibtex) > 0: - bibtex_name = list(bibtex.keys())[0] - bibtex = bibtex[bibtex_name] - bibtex_string = tools.parsed2Bibtex(bibtex) - else: - bibtex_string = '' - - if not autoconfirm: - bibtex = checkBibtex(src, bibtex_string) - - if not autoconfirm: - tag = tools.rawInput("Tag for this paper (leave empty for default) ? ") - else: - tag = args.tag - bibtex['tag'] = tag - - if rename: - new_name = backend.getNewName(src, bibtex, tag) - - while os.path.exists(new_name): - tools.warning("file "+new_name+" already exists.") - default_rename = new_name.replace(tools.getExtension(new_name), - " (2)" + - tools.getExtension(new_name)) - rename = tools.rawInput("New name ["+default_rename+"]? ") - if rename == '': - new_name = default_rename - else: - new_name = rename - try: - shutil.copy2(src, new_name) - except shutil.Error: - new_name = False - sys.exit("Unable to move file to library dir " + - config.get("folder")+".") - else: - new_name = src - bibtex['file'] = os.path.abspath(new_name) - - # Remove first page of IOP papers - try: - if 'IOP' in bibtex['publisher'] and bibtex['ENTRYTYPE'] == 'article': - tearpages.tearpage(new_name) - except (KeyError, shutil.Error, IOError): - pass - - backend.bibtexAppend(bibtex) - return new_name - - -def editEntry(entry, file_id='both'): - bibtex = backend.getBibtex(entry, file_id) - if bibtex is False: - tools.warning("Entry "+entry+" does not exist.") - return False - - if file_id == 'file': - filename = entry - else: - filename = bibtex['file'] - new_bibtex = checkBibtex(filename, tools.parsed2Bibtex(bibtex)) - - # Tag update - if new_bibtex['tag'] != bibtex['tag']: - print("Editing tag, moving file.") - new_name = backend.getNewName(new_bibtex['file'], - new_bibtex, - new_bibtex['tag']) - - while os.path.exists(new_name): - tools.warning("file "+new_name+" already exists.") - default_rename = new_name.replace(tools.getExtension(new_name), - " (2)" + - tools.getExtension(new_name)) - rename = tools.rawInput("New name ["+default_rename+"]? ") - if rename == '': - new_name = default_rename - else: - new_name = rename - new_bibtex['file'] = new_name - - try: - shutil.move(bibtex['file'], new_bibtex['file']) - except shutil.Error: - tools.warning('Unable to move file '+bibtex['file']+' to ' + - new_bibtex['file'] + ' according to tag edit.') - - try: - if not os.listdir(os.path.dirname(bibtex['file'])): - os.rmdir(os.path.dirname(bibtex['file'])) - except OSError: - tools.warning("Unable to delete empty tag dir " + - os.path.dirname(bibtex['file'])) - - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - index = bibtexparser.load(fh) - index = index.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - index[new_bibtex['ID']] = new_bibtex - backend.bibtexRewrite(index) - return True - - -def downloadFile(url, filetype, manual, autoconfirm, tag): - print('Downloading '+url) - dl, contenttype = fetcher.download(url) - - if dl is not False: - print('Download finished') - tmp = tempfile.NamedTemporaryFile(suffix='.'+contenttype) - - with open(tmp.name, 'wb+') as fh: - fh.write(dl) - new_name = addFile(tmp.name, filetype, manual, autoconfirm, tag) - if new_name is False: - return False - tmp.close() - return new_name - else: - tools.warning("Could not fetch "+url) - return False - - -def openFile(ident): - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - if ident not in list(bibtex.keys()): - return False - else: - subprocess.Popen(['xdg-open', bibtex[ident]['file']]) - return True - - -def resync(): - diff = backend.diffFilesIndex() - - if diff is False: - return False - - for key in diff: - entry = diff[key] - if entry['file'] == '': - print("\nFound entry in index without associated file: " + - entry['ID']) - print("Title:\t"+entry['title']) - loop = True - while confirm: - filename = tools.rawInput("File to import for this entry " + - "(leave empty to delete the " + - "entry)? ") - if filename == '': - break - else: - if 'doi' in list(entry.keys()): - doi = fetcher.findArticleID(filename, only=["DOI"]) - if doi is not False and doi != entry['doi']: - loop = tools.rawInput("Found DOI does not " + - "match bibtex entry " + - "DOI, continue anyway " + - "? [y/N]") - loop = (loop.lower() != 'y') - if 'Eprint' in list(entry.keys()): - arxiv = fetcher.findArticleID(filename, only=["arXiv"]) - if arxiv is not False and arxiv != entry['Eprint']: - loop = tools.rawInput("Found arXiv id does " + - "not match bibtex " + - "entry arxiv id, " + - "continue anyway ? [y/N]") - loop = (loop.lower() != 'y') - if 'isbn' in list(entry.keys()): - isbn = fetcher.findISBN(filename) - if isbn is not False and isbn != entry['isbn']: - loop = tools.rawInput("Found ISBN does not " + - "match bibtex entry " + - "ISBN, continue anyway " + - "? [y/N]") - loop = (loop.lower() != 'y') - continue - if filename == '': - backend.deleteId(entry['ID']) - print("Deleted entry \""+entry['ID']+"\".") - else: - new_name = backend.getNewName(filename, entry) - try: - shutil.copy2(filename, new_name) - print("Imported new file "+filename+" for entry " + - entry['ID']+".") - except shutil.Error: - new_name = False - sys.exit("Unable to move file to library dir " + - config.get("folder")+".") - backend.bibtexEdit(entry['ID'], {'file': filename}) - else: - print("Found file without any associated entry in index:") - print(entry['file']) - action = '' - while action.lower() not in ['import', 'delete']: - action = tools.rawInput("What to do? [import / delete] ") - action = action.lower() - if action == 'import': - tmp = tempfile.NamedTemporaryFile() - shutil.copy(entry['file'], tmp.name) - filetype = tools.getExtension(entry['file']) - try: - os.remove(entry['file']) - except OSError: - tools.warning("Unable to delete file "+entry['file']) - if not addFile(tmp.name, filetype): - tools.warning("Unable to reimport file "+entry['file']) - tmp.close() - else: - backend.deleteFile(entry['file']) - print(entry['file'] + " removed from disk and " + - "index.") - # Check for empty tag dirs - for i in os.listdir(config.get("folder")): - if os.path.isdir(i) and not os.listdir(config.get("folder") + i): - try: - os.rmdir(config.get("folder") + i) - except OSError: - tools.warning("Found empty tag dir "+config.get("folder") + i + - " but could not delete it.") - - -def update(entry): - update = backend.updateArXiv(entry) - if update is not False: - print("New version found for "+entry) - print("\t Title: "+update['title']) - confirm = tools.rawInput("Download it ? [Y/n] ") - if confirm.lower() == 'n': - return - new_name = downloadFile('http://arxiv.org/pdf/'+update['eprint'], - 'article', False) - if new_name is not False: - print(update['eprint']+" successfully imported as "+new_name) - else: - tools.warning("An error occurred while downloading "+url) - confirm = tools.rawInput("Delete previous version ? [y/N] ") - if confirm.lower() == 'y': - if not backend.deleteId(entry): - if not backend.deleteFile(entry): - tools.warning("Unable to remove previous version.") - return - print("Previous version successfully deleted.") - - -def commandline_arg(bytestring): - # UTF-8 encoding for python2 - if sys.version_info >= (3, 0): - unicode_string = bytestring - else: - unicode_string = bytestring.decode(sys.getfilesystemencoding()) - return unicode_string - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="A bibliography " + - "management tool.") + parser = argparse.ArgumentParser( + description="A bibliography management tool.") + parser.add_argument("-c", "--config", default=None, + help="path to a custom config dir.") subparsers = parser.add_subparsers(help="sub-command help", dest='parser') subparsers.required = True # Fix for Python 3.3.5 parser_download = subparsers.add_parser('download', help="download help") parser_download.add_argument('-t', '--type', default=None, - choices=['article', 'book'], - help="type of the file to download", - type=commandline_arg) + choices=['paper', 'book'], + help="type of the file to download") parser_download.add_argument('-m', '--manual', default=False, action='store_true', help="disable auto-download of bibtex") parser_download.add_argument('-y', default=False, help="Confirm all") parser_download.add_argument('--tag', default='', - help="Tag", type=commandline_arg) + help="Tag") parser_download.add_argument('--keep', default=False, help="Do not remove the file") parser_download.add_argument('url', nargs='+', - help="url of the file to import", - type=commandline_arg) + help="url of the file to import") parser_download.set_defaults(func='download') parser_import = subparsers.add_parser('import', help="import help") parser_import.add_argument('-t', '--type', default=None, - choices=['article', 'book'], - help="type of the file to import", - type=commandline_arg) + choices=['paper', 'book'], + help="type of the file to import") parser_import.add_argument('-m', '--manual', default=False, action='store_true', help="disable auto-download of bibtex") parser_import.add_argument('-y', default=False, help="Confirm all") - parser_import.add_argument('--tag', default='', help="Tag", - type=commandline_arg) + parser_import.add_argument('--tag', default='', help="Tag") parser_import.add_argument('--in-place', default=False, dest="inplace", action='store_true', help="Leave the imported file in place",) parser_import.add_argument('file', nargs='+', - help="path to the file to import", - type=commandline_arg) + help="path to the file to import") parser_import.add_argument('--skip', nargs='+', - help="path to files to skip", default=[], - type=commandline_arg) + help="path to files to skip", default=[]) parser_import.set_defaults(func='import') parser_delete = subparsers.add_parser('delete', help="delete help") parser_delete.add_argument('entries', metavar='entry', nargs='+', - help="a filename or an identifier", - type=commandline_arg) + help="a filename or an identifier") parser_delete.add_argument('--skip', nargs='+', - help="path to files to skip", default=[], - type=commandline_arg) + help="path to files to skip", default=[]) group = parser_delete.add_mutually_exclusive_group() group.add_argument('--id', action="store_true", default=False, help="id based deletion") @@ -514,11 +81,9 @@ if __name__ == '__main__': parser_edit = subparsers.add_parser('edit', help="edit help") parser_edit.add_argument('entries', metavar='entry', nargs='+', - help="a filename or an identifier", - type=commandline_arg) + help="a filename or an identifier") parser_edit.add_argument('--skip', nargs='+', - help="path to files to skip", default=[], - type=commandline_arg) + help="path to files to skip", default=[]) group = parser_edit.add_mutually_exclusive_group() group.add_argument('--id', action="store_true", default=False, help="id based deletion") @@ -529,19 +94,14 @@ if __name__ == '__main__': parser_list = subparsers.add_parser('list', help="list help") parser_list.set_defaults(func='list') - parser_search = subparsers.add_parser('search', help="search help") - parser_search.set_defaults(func='search') - parser_open = subparsers.add_parser('open', help="open help") parser_open.add_argument('ids', metavar='id', nargs='+', - help="an identifier", - type=commandline_arg) + help="an identifier") parser_open.set_defaults(func='open') parser_export = subparsers.add_parser('export', help="export help") parser_export.add_argument('ids', metavar='id', nargs='+', - help="an identifier", - type=commandline_arg) + help="an identifier") parser_export.set_defaults(func='export') parser_resync = subparsers.add_parser('resync', help="resync help") @@ -549,131 +109,133 @@ if __name__ == '__main__': parser_update = subparsers.add_parser('update', help="update help") parser_update.add_argument('--entries', metavar='entry', nargs='+', - help="a filename or an identifier", - type=commandline_arg) + help="a filename or an identifier") parser_update.set_defaults(func='update') - parser_search = subparsers.add_parser('search', help="search help") - parser_search.add_argument('query', metavar='entry', nargs='+', - help="your query, see README for more info.", - type=commandline_arg) - parser_search.set_defaults(func='search') + return parser.parse_args() - args = parser.parse_args() - try: - if args.func == 'download': - skipped = [] - for url in args.url: - new_name = downloadFile(url, args.type, args.manual, args.y, - args.tag) - if new_name is not False: - print(url+" successfully imported as "+new_name) - else: - tools.warning("An error occurred while downloading "+url) - skipped.append(url) - if len(skipped) > 0: - print("\nSkipped files:") - for i in skipped: - print(i) - sys.exit() - if args.func == 'import': - skipped = [] - for filename in list(set(args.file) - set(args.skip)): - new_name = addFile(filename, args.type, args.manual, args.y, - args.tag, not args.inplace) - if new_name is not False: - print(filename+" successfully imported as " + - new_name+".") - else: - tools.warning("An error occurred while importing " + - filename) - skipped.append(filename) - if len(skipped) > 0: - print("\nSkipped files:") - for i in skipped: - print(i) - sys.exit() +def main(): + """ + Main function. + """ + global config - elif args.func == 'delete': - skipped = [] - for filename in list(set(args.entries) - set(args.skip)): - if not args.force: - confirm = tools.rawInput("Are you sure you want to " + - "delete "+filename+" ? [y/N] ") - else: - confirm = 'y' + # Parse arguments + args = parse_args() - if confirm.lower() == 'y': - if args.file or not backend.deleteId(filename, args.keep): - if(args.id or - not backend.deleteFile(filename, args.keep)): - tools.warning("Unable to delete "+filename) - sys.exit(1) + # Load the custom config if needed + if args.config is not None: + config = Config(base_config_path=args.config) - print(filename+" successfully deleted.") - else: - skipped.append(filename) - - if len(skipped) > 0: - print("\nSkipped files:") - for i in skipped: - print(i) - sys.exit() - - elif args.func == 'edit': - for filename in list(set(args.entries) - set(args.skip)): - if args.file: - file_id = 'file' - elif args.id: - file_id = 'id' - else: - file_id = 'both' - editEntry(filename, file_id) - sys.exit() - - elif args.func == 'list': - listPapers = backend.getEntries(full=True) - if not listPapers: - sys.exit() - listPapers = [v["file"] for k, v in listPapers.items()] - listPapers.sort() - for paper in listPapers: - print(paper) - sys.exit() - - elif args.func == 'search': - raise Exception('TODO') - - elif args.func == 'open': - for filename in args.ids: - if not openFile(filename): - sys.exit("Unable to open file associated " + - "to ident "+filename) - sys.exit() - - elif args.func == 'export': - bibtex = '' - for id in args.ids: - bibtex += tools.parsed2Bibtex(backend.getBibtex(id, - clean=True)) - print(bibtex.strip()) - sys.exit - - elif args.func == 'resync': - confirm = tools.rawInput("Resync files and bibtex index? [y/N] ") - if confirm.lower() == 'y': - resync() - sys.exit() - - elif args.func == 'update': - if args.entries is None: - entries = backend.getEntries() + # Download command + if args.func == 'download': + skipped = [] + for url in args.url: + # Try to download the URL + new_name = commands.download(url, args.type, args.manual, args.y, + args.tag) + if new_name is not None: + print("%s successfully imported as %s." % (url, new_name)) else: - entries = args.entries - for entry in entries: - update(entry) - sys.exit() + tools.warning("An error occurred while downloading %s." % + (url,)) + skipped.append(url) + # Output URLs with errors + if len(skipped) > 0: + tools.warning("Skipped URLs:") + for i in skipped: + tools.warning(i) + # Import command + elif args.func == 'import': + skipped = [] + # Handle exclusions + files_to_process = list(set(args.file) - set(args.skip)) + for filename in files_to_process: + # Try to import the file + new_name = commands.import_file(filename, args.type, + args.manual, args.y, + args.tag, not args.inplace) + if new_name is not None: + print("%s successfully imported as %s." % (filename, new_name)) + else: + tools.warning("An error occurred while importing %s." % + (filename,)) + skipped.append(filename) + # Output files with errors + if len(skipped) > 0: + tools.warning("Skipped files:") + for i in skipped: + tools.warning(i) + + # Delete command + elif args.func == 'delete': + skipped = [] + # Handle exclusions + items_to_process = list(set(args.entries) - set(args.skip)) + for item in items_to_process: + # Confirm before deletion + if not args.force: + confirm = input("Are you sure you want to delete %s? [y/N] " % + (item,)) + else: + confirm = 'y' + + # Try to delete the item + if confirm.lower() == 'y': + file_or_id = "id" if args.id else "file" if args.file else None + commands.delete(item, args.keep, file_or_id) + print("%s successfully deleted." % (item,)) + else: + skipped.append(item) + # Output items with errors + if len(skipped) > 0: + tools.warning("Skipped items:") + for i in skipped: + tools.warning(i) + + # Edit command + elif args.func == 'edit': + # Handle exclusions + items_to_process = list(set(args.entries) - set(args.skip)) + for item in items_to_process: + file_or_id = "id" if args.id else "file" if args.file else None + commands.edit(item, file_or_id) + + # List command + elif args.func == 'list': + # List all available items + for id, file in commands.list().items(): + # And print them as "identifier: file" + print("%s: %s" % (id, file)) + + # Open command + elif args.func == 'open': + # Open each entry + for id in args.ids: + if commands.open(id) is None: + # And warn the user about missing files or errors + tools.warning("Unable to open file associated with ident %s." % + (id,)) + + # Export command + elif args.func == 'export': + print(commands.export(args.ids)) + + # Resync command + elif args.func == 'resync': + confirm = input("Resync files and bibtex index? [y/N] ") + if confirm.lower() == 'y': + commands.resync() + + # Update command + elif args.func == 'update': + commands.update(args.entries) + + +if __name__ == '__main__': + try: + main() except KeyboardInterrupt: sys.exit() diff --git a/libbmc/__init__.py b/libbmc/__init__.py deleted file mode 100644 index aec8850..0000000 --- a/libbmc/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- diff --git a/libbmc/backend.py b/libbmc/backend.py deleted file mode 100644 index 365b034..0000000 --- a/libbmc/backend.py +++ /dev/null @@ -1,336 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- - -from __future__ import unicode_literals -import os -import re -import libbmc.tools as tools -import libbmc.fetcher as fetcher -import bibtexparser -from libbmc.config import Config -from codecs import open - - -config = Config() - - -def getNewName(src, bibtex, tag='', override_format=None): - """ - Return the formatted name according to config for the given - bibtex entry - """ - authors = re.split(' and ', bibtex['author']) - - if bibtex['ENTRYTYPE'] == 'article': - if override_format is None: - new_name = config.get("format_articles") - else: - new_name = override_format - try: - new_name = new_name.replace("%j", bibtex['journal']) - except KeyError: - pass - elif bibtex['ENTRYTYPE'] == 'book': - if override_format is None: - new_name = config.get("format_books") - else: - new_name = override_format - - new_name = new_name.replace("%t", bibtex['title']) - try: - new_name = new_name.replace("%Y", bibtex['year']) - except KeyError: - pass - new_name = new_name.replace("%f", authors[0].split(',')[0].strip()) - new_name = new_name.replace("%l", authors[-1].split(',')[0].strip()) - new_name = new_name.replace("%a", ', '.join([i.split(',')[0].strip() - for i in authors])) - if('archiveprefix' in bibtex and - 'arXiv' in bibtex['archiveprefix']): - new_name = new_name.replace("%v", - '-' + - bibtex['eprint'][bibtex['eprint']. - rfind('v'):]) - else: - new_name = new_name.replace("%v", '') - - for custom in config.get("format_custom"): - new_name = custom(new_name) - - if tag == '': - new_name = (config.get("folder") + tools.slugify(new_name) + - tools.getExtension(src)) - else: - if not os.path.isdir(config.get("folder") + tag): - try: - os.mkdir(config.get("folder") + tag) - except OSError: - tools.warning("Unable to create tag dir " + - config.get("folder")+tag+".") - - new_name = (config.get("folder") + tools.slugify(tag) + '/' + - tools.slugify(new_name) + tools.getExtension(src)) - - return new_name - - -def bibtexAppend(data): - """Append data to the main bibtex file - - data is a dict for one entry in bibtex, as the one from bibtexparser output - """ - try: - with open(config.get("folder")+'index.bib', 'a', encoding='utf-8') \ - as fh: - fh.write(tools.parsed2Bibtex(data)+"\n") - except IOError as e: - raise e - tools.warning("Unable to open index file.") - return False - - -def bibtexEdit(ident, modifs): - """Update ident key in bibtex file, modifications are in modifs dict""" - - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (IOError, TypeError): - tools.warning("Unable to open index file.") - return False - - for key in modifs.keys(): - bibtex[ident][key] = modifs[key] - bibtexRewrite(bibtex) - - -def bibtexRewrite(data): - """Rewrite the bibtex index file. - - data is a dict of bibtex entry dict. - """ - bibtex = '' - for entry in data.keys(): - bibtex += tools.parsed2Bibtex(data[entry])+"\n" - try: - with open(config.get("folder")+'index.bib', 'w', encoding='utf-8') \ - as fh: - fh.write(bibtex) - except (IOError, TypeError): - tools.warning("Unable to open index file.") - return False - - -def deleteId(ident, keep=False): - """Delete a file based on its id in the bibtex file""" - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (IOError, TypeError): - tools.warning("Unable to open index file.") - return False - - if ident not in bibtex.keys(): - return False - - if not keep: - try: - os.remove(bibtex[ident]['file']) - except (KeyError, OSError): - tools.warning("Unable to delete file associated to id " + ident + - " : " + bibtex[ident]['file']) - - try: - if not os.listdir(os.path.dirname(bibtex[ident]['file'])): - os.rmdir(os.path.dirname(bibtex[ident]['file'])) - except (KeyError, OSError): - tools.warning("Unable to delete empty tag dir " + - os.path.dirname(bibtex[ident]['file'])) - - try: - del(bibtex[ident]) - bibtexRewrite(bibtex) - except KeyError: - tools.warning("No associated bibtex entry in index for file " + - bibtex[ident]['file']) - return True - - -def deleteFile(filename, keep=False): - """Delete a file based on its filename""" - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - found = False - for key in list(bibtex.keys()): - try: - if os.path.samefile(bibtex[key]['file'], filename): - found = True - if not keep: - try: - os.remove(bibtex[key]['file']) - except (KeyError, OSError): - tools.warning("Unable to delete file associated " + - "to id " + key+" : "+bibtex[key]['file']) - - try: - if not os.listdir(os.path.dirname(filename)): - os.rmdir(os.path.dirname(filename)) - except OSError: - tools.warning("Unable to delete empty tag dir " + - os.path.dirname(filename)) - - try: - del(bibtex[key]) - except KeyError: - tools.warning("No associated bibtex entry in index for " + - "file " + bibtex[key]['file']) - except (KeyError, OSError): - pass - if found: - bibtexRewrite(bibtex) - elif os.path.isfile(filename): - os.remove(filename) - return found - - -def diffFilesIndex(): - """Compute differences between Bibtex index and PDF files - - Returns a dict with bibtex entry: - * full bibtex entry with file='' if file is not found - * only file entry if file with missing bibtex entry - """ - files = tools.listDir(config.get("folder")) - files = [i for i in files if tools.getExtension(i) in ['.pdf', '.djvu']] - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - index = bibtexparser.load(fh) - index_diff = index.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - for key in index_diff.keys(): - if index_diff[key]['file'] not in files: - index_diff[key]['file'] = '' - else: - files.remove(index_diff[key]['file']) - - for filename in files: - index_diff[filename] = {'file': filename} - - return index.entries_dict - - -def getBibtex(entry, file_id='both', clean=False): - """Returns the bibtex entry corresponding to entry, as a dict - - entry is either a filename or a bibtex ident - file_id is file or id or both to search for a file / id / both - clean is to clean the ignored fields specified in config - """ - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - bibtex_entry = False - if file_id == 'both' or file_id == 'id': - try: - bibtex_entry = bibtex[entry] - except KeyError: - pass - if file_id == 'both' or file_id == 'file': - if os.path.isfile(entry): - for key in bibtex.keys(): - if os.path.samefile(bibtex[key]['file'], entry): - bibtex_entry = bibtex[key] - break - if clean: - for field in config.get("ignore_fields"): - try: - del(bibtex_entry[field]) - except KeyError: - pass - return bibtex_entry - - -def getEntries(full=False): - """Returns the list of all entries in the bibtex index""" - try: - with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \ - as fh: - bibtex = bibtexparser.load(fh) - bibtex = bibtex.entries_dict - except (TypeError, IOError): - tools.warning("Unable to open index file.") - return False - - if full: - return bibtex - else: - return list(bibtex.keys()) - - -def updateArXiv(entry): - """Look for new versions of arXiv entry `entry` - - Returns False if no new versions or not an arXiv entry, - Returns the new bibtex otherwise. - """ - bibtex = getBibtex(entry) - # Check arXiv - if('archiveprefix' not in bibtex or - 'arXiv' not in bibtex['archiveprefix']): - return False - - arxiv_id = bibtex['eprint'] - arxiv_id_no_v = re.sub(r'v\d+\Z', '', arxiv_id) - ids = set(arxiv_id) - - for entry in getEntries(): - if('archiveprefix' not in bibtex or - 'arXiv' not in bibtex['archiveprefix']): - continue - ids.add(bibtex['eprint']) - - last_bibtex = bibtexparser.loads(fetcher.arXiv2Bib(arxiv_id_no_v)) - last_bibtex = last_bibtex.entries_dict - last_bibtex = last_bibtex[list(last_bibtex.keys())[0]] - - if last_bibtex['eprint'] not in ids: - return last_bibtex - else: - return False - - -def search(query): - """Performs a search in the bibtex index. - - Param: query is a dict of keys and the query for these keys - """ - raise Exception('TODO') diff --git a/libbmc/fetcher.py b/libbmc/fetcher.py deleted file mode 100644 index 29fb716..0000000 --- a/libbmc/fetcher.py +++ /dev/null @@ -1,359 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- - - -import isbnlib -import re -import socket -import socks -import subprocess -import sys -try: - # For Python 3.0 and later - from urllib.request import urlopen, Request - from urllib.error import URLError -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, Request, URLError -import arxiv2bib as arxiv_metadata -import libbmc.tools as tools -import bibtexparser -from libbmc.config import Config - - -config = Config() -default_socket = socket.socket -try: - stdout_encoding = sys.stdout.encoding - assert(stdout_encoding is not None) -except (AttributeError, AssertionError): - stdout_encoding = 'UTF-8' - - -def download(url): - """Download url tofile - - Check that it is a valid pdf or djvu file. Tries all the - available proxies sequentially. Returns the raw content of the file, or - false if it could not be downloaded. - """ - for proxy in config.get("proxies"): - if proxy.startswith('socks'): - if proxy[5] == '4': - proxy_type = socks.SOCKS4 - else: - proxy_type = socks.SOCKS5 - proxy = proxy[proxy.find('://')+3:] - try: - proxy, port = proxy.split(':') - except ValueError: - port = None - socks.set_default_proxy(proxy_type, proxy, port) - socket.socket = socks.socksocket - elif proxy == '': - socket.socket = default_socket - else: - try: - proxy, port = proxy.split(':') - except ValueError: - port = None - socks.set_default_proxy(socks.HTTP, proxy, port) - socket.socket = socks.socksocket - try: - r = urlopen(url) - try: - size = int(dict(r.info())['content-length'].strip()) - except KeyError: - try: - size = int(dict(r.info())['Content-Length'].strip()) - except KeyError: - size = 0 - dl = b"" - dl_size = 0 - while True: - buf = r.read(1024) - if buf: - dl += buf - dl_size += len(buf) - if size != 0: - done = int(50 * dl_size / size) - sys.stdout.write("\r[%s%s]" % ('='*done, ' '*(50-done))) - sys.stdout.write(" "+str(int(float(done)/52*100))+"%") - sys.stdout.flush() - else: - break - contenttype = False - contenttype_req = None - try: - contenttype_req = dict(r.info())['content-type'] - except KeyError: - try: - contenttype_req = dict(r.info())['Content-Type'] - except KeyError: - continue - try: - if 'pdf' in contenttype_req: - contenttype = 'pdf' - elif 'djvu' in contenttype_req: - contenttype = 'djvu' - except KeyError: - pass - - if r.getcode() != 200 or contenttype is False: - continue - - return dl, contenttype - except ValueError: - tools.warning("Invalid URL") - return False, None - except (URLError, socket.error): - if proxy != "": - proxy_txt = "using proxy "+proxy - else: - proxy_txt = "without using any proxy" - tools.warning("Unable to get "+url+" "+proxy_txt+". It " + - "may not be available at the moment.") - continue - return False, None - - -isbn_re = re.compile(r'isbn[\s]?:?[\s]?((?:[0-9]{3}[ -]?)?[0-9]{1,5}[ -]?[0-9]{1,7}[ -]?[0-9]{1,6}[- ]?[0-9])', - re.IGNORECASE) - - -def findISBN(src): - """Search for a valid ISBN in src. - - Returns the ISBN or false if not found or an error occurred.""" - if src.endswith(".pdf"): - totext = subprocess.Popen(["pdftotext", src, "-"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1) - elif src.endswith(".djvu"): - totext = subprocess.Popen(["djvutxt", src], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1) - else: - return False - - while totext.poll() is None: - extractfull = ' '.join([i.decode(stdout_encoding).strip() for i in totext.stdout.readlines()]) - extractISBN = isbn_re.search(extractfull.lower().replace('Œ', - '-')) - if extractISBN: - totext.terminate() - break - - err = totext.communicate()[1] - if totext.returncode > 0: - # Error happened - tools.warning(err) - return False - - cleanISBN = False - # Clean ISBN is the ISBN number without separators - if extractISBN: - cleanISBN = extractISBN.group(1).replace('-', '').replace(' ', '') - return cleanISBN - - -def isbn2Bib(isbn): - """Tries to get bibtex entry from an ISBN number""" - # Default merges results from worldcat.org and google books - try: - return isbnlib.registry.bibformatters['bibtex'](isbnlib.meta(isbn, - 'default')) - except (isbnlib.ISBNLibException, TypeError): - return '' - - -doi_re = re.compile('(?<=doi)/?:?\s?[0-9\.]{7}/\S*[0-9]', re.IGNORECASE) -doi_pnas_re = re.compile('(?<=doi).?10.1073/pnas\.\d+', re.IGNORECASE) -doi_jsb_re = re.compile('10\.1083/jcb\.\d{9}', re.IGNORECASE) -clean_doi_re = re.compile('^/') -clean_doi_fabse_re = re.compile('^10.1096') -clean_doi_jcb_re = re.compile('^10.1083') -clean_doi_len_re = re.compile(r'\d\.\d') -arXiv_re = re.compile(r'arXiv:\s*([\w\.\/\-]+)', re.IGNORECASE) - - -def findArticleID(src, only=["DOI", "arXiv"]): - """Search for a valid article ID (DOI or ArXiv) in src. - - Returns a tuple (type, first matching ID) or False if not found - or an error occurred. - From : http://en.dogeno.us/2010/02/release-a-python-script-for-organizing-scientific-papers-pyrenamepdf-py/ - and https://github.com/minad/bibsync/blob/3fdf121016f6187a2fffc66a73cd33b45a20e55d/lib/bibsync/utils.rb - """ - if src.endswith(".pdf"): - totext = subprocess.Popen(["pdftotext", src, "-"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - elif src.endswith(".djvu"): - totext = subprocess.Popen(["djvutxt", src], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - return (False, False) - - extractfull = '' - extract_type = False - extractID = None - while totext.poll() is None: - extractfull += ' '.join([i.decode(stdout_encoding).strip() for i in totext.stdout.readlines()]) - # Try to extract DOI - if "DOI" in only: - extractlower = extractfull.lower().replace('digital object identifier', 'doi') - extractID = doi_re.search(extractlower.replace('Œ', '-')) - if not extractID: - # PNAS fix - extractID = doi_pnas_re.search(extractlower.replace('pnas', '/pnas')) - if not extractID: - # JSB fix - extractID = doi_jsb_re.search(extractlower) - if extractID: - extract_type = "DOI" - totext.terminate() - # Try to extract arXiv - if "arXiv" in only: - tmp_extractID = arXiv_re.search(extractfull) - if tmp_extractID: - if not extractID or extractID.start(0) > tmp_extractID.start(1): - # Only use arXiv id if it is before the DOI in the pdf - extractID = tmp_extractID - extract_type = "arXiv" - totext.terminate() - if extract_type is not False: - break - - err = totext.communicate()[1] - if totext.returncode > 0: - # Error happened - tools.warning(err) - return (False, False) - - if extractID is not None and extract_type == "DOI": - # If DOI extracted, clean it and return it - cleanDOI = False - cleanDOI = extractID.group(0).replace(':', '').replace(' ', '') - if clean_doi_re.search(cleanDOI): - cleanDOI = cleanDOI[1:] - # FABSE J fix - if clean_doi_fabse_re.search(cleanDOI): - cleanDOI = cleanDOI[:20] - # Second JCB fix - if clean_doi_jcb_re.search(cleanDOI): - cleanDOI = cleanDOI[:21] - if len(cleanDOI) > 40: - cleanDOItemp = clean_doi_len_re.sub('000', cleanDOI) - reps = {'.': 'A', '-': '0'} - cleanDOItemp = tools.replaceAll(cleanDOItemp[8:], reps) - digitStart = 0 - for i in range(len(cleanDOItemp)): - if cleanDOItemp[i].isdigit(): - digitStart = 1 - if cleanDOItemp[i].isalpha() and digitStart: - break - cleanDOI = cleanDOI[0:(8+i)] - return ("DOI", cleanDOI) - elif extractID is not None and extract_type == "arXiv": - # If arXiv id is extracted, return it - return ("arXiv", extractID.group(1)) - return (False, False) - - -def doi2Bib(doi): - """Returns a bibTeX string of metadata for a given DOI. - - From : https://gist.github.com/jrsmith3/5513926 - """ - url = "http://dx.doi.org/" + doi - headers = {"accept": "application/x-bibtex"} - req = Request(url, headers=headers) - try: - r = urlopen(req) - - try: - if dict(r.info())['content-type'] == 'application/x-bibtex': - return r.read().decode('utf-8') - else: - return '' - except KeyError: - try: - if dict(r.info())['Content-Type'] == 'application/x-bibtex': - return r.read().decode('utf-8') - else: - return '' - except KeyError: - return '' - except: - tools.warning('Unable to contact remote server to get the bibtex ' + - 'entry for doi '+doi) - return '' - - -def arXiv2Bib(arxiv): - """Returns bibTeX string of metadata for a given arXiv id - - arxiv is an arxiv id - """ - bibtex = arxiv_metadata.arxiv2bib([arxiv]) - for bib in bibtex: - if isinstance(bib, arxiv_metadata.ReferenceErrorInfo): - continue - else: - fetched_bibtex = bibtexparser.loads(bib.bibtex()) - fetched_bibtex = fetched_bibtex.entries_dict - fetched_bibtex = fetched_bibtex[list(fetched_bibtex.keys())[0]] - try: - del(fetched_bibtex['file']) - except KeyError: - pass - return tools.parsed2Bibtex(fetched_bibtex) - return '' - - -HAL_re = re.compile(r'(hal-\d{8}), version (\d+)') - - -def findHALId(src): - """Searches for a valid HAL id in src - - Returns a tuple of the HAL id and the version - or False if not found or an error occurred. - """ - if src.endswith(".pdf"): - totext = subprocess.Popen(["pdftotext", src, "-"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - elif src.endswith(".djvu"): - totext = subprocess.Popen(["djvutxt", src], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - return False - - while totext.poll() is None: - extractfull = ' '.join([i.decode(stdout_encoding).strip() for i in totext.stdout.readlines()]) - extractID = HAL_re.search(extractfull) - if extractID: - totext.terminate() - break - - err = totext.communicate()[1] - if totext.returncode > 0: - # Error happened - tools.warning(err) - return False - else: - return extractID.group(1), extractID.group(2) diff --git a/libbmc/search.py b/libbmc/search.py deleted file mode 100644 index eb132f4..0000000 --- a/libbmc/search.py +++ /dev/null @@ -1,298 +0,0 @@ -# -*- coding: utf8 -*- -"""Search query parser - -Modified by Phyks, 2014-05-18. Original source code is here: -http://pyparsing.wikispaces.com/file/view/searchparser.py/30112816/searchparser.py - -version 2006-03-09 - -This search query parser uses the excellent Pyparsing module -(http://pyparsing.sourceforge.net/) to parse search queries by users. -It handles: - -* 'and', 'or' and implicit 'and' operators; -* parentheses; -* quoted strings; -* wildcards at the end of a search term (help*); - -Requirements: -* Python -* Pyparsing - -If you run this script, it will perform a number of tests. To use is as a -module, you should use inheritance on the SearchQueryParser class and overwrite -the Get... methods. The ParserTest class gives a very simple example of how this -could work. - -------------------------------------------------------------------------------- -Copyright (c) 2006, Estrate, the Netherlands -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. -* Neither the name of Estrate nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -CONTRIBUTORS: -- Steven Mooij -- Rudolph Froger -- Paul McGuire -""" -from pyparsing import Word, alphanums, Keyword, Group, Combine, Forward -from pyparsing import Suppress, Optional, OneOrMore, oneOf, Or - -class SearchQueryParser: - - def __init__(self): - self._methods = { - 'and': self.evaluateAnd, - 'or': self.evaluateOr, - 'not': self.evaluateNot, - 'parenthesis': self.evaluateParenthesis, - 'quotes': self.evaluateQuotes, - 'word': self.evaluateWord, - 'wordwildcard': self.evaluateWordWildcard, - } - self._parser = self.parser() - - def parser(self): - """ - This function returns a parser. - The grammar should be like most full text search engines (Google, Tsearch, Lucene). - - Grammar: - - a query consists of alphanumeric words, with an optional '*' wildcard - at the end of a word - - a sequence of words between quotes is a literal string - - words can be used together by using operators ('and' or 'or') - - words with operators can be grouped with parenthesis - - a word or group of words can be preceded by a 'not' operator - - the 'and' operator precedes an 'or' operator - - if an operator is missing, use an 'and' operator - """ - operatorOr = Forward() - - operatorWord = Group(Combine(Word(alphanums) + Suppress('*'))).setResultsName('wordwildcard') | \ - Group(Word(alphanums)).setResultsName('word') - - operatorQuotesContent = Forward() - operatorQuotesContent << ( - (operatorWord + operatorQuotesContent) | operatorWord - ) - - operatorQuotes = Group( - Or([Suppress('"') + operatorQuotesContent + Suppress('"'), - Suppress('\'') + operatorQuotesContent + Suppress('\'')] - )).setResultsName("quotes") | operatorWord - - operatorParenthesis = Group( - (Suppress("(") + operatorOr + Suppress(")")) - ).setResultsName("parenthesis") | operatorQuotes - - operatorNot = Forward() - operatorNot << (Group( - Suppress(Keyword("not", caseless=True)) + operatorNot - ).setResultsName("not") | operatorParenthesis) - - operatorAnd = Forward() - operatorAnd << (Group( - operatorNot + Suppress(Keyword("and", caseless=True)) + operatorAnd - ).setResultsName("and") | Group( - operatorNot + OneOrMore(~oneOf("and or") + operatorAnd) - ).setResultsName("and") | operatorNot) - - operatorOr << (Group( - operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr - ).setResultsName("or") | operatorAnd) - - operatorQ = Forward() - operatorQ << Group(operatorOr + Suppress('=') + - operatorOr).setResultsName('field') - - return operatorQ.parseString - - def evaluateAnd(self, argument): - return self.evaluate(argument[0]).intersection(self.evaluate(argument[1])) - - def evaluateOr(self, argument): - return self.evaluate(argument[0]).union(self.evaluate(argument[1])) - - def evaluateNot(self, argument): - return self.GetNot(self.evaluate(argument[0])) - - def evaluateParenthesis(self, argument): - return self.evaluate(argument[0]) - - def evaluateQuotes(self, argument): - """Evaluate quoted strings - - First is does an 'and' on the indidual search terms, then it asks the - function GetQuoted to only return the subset of ID's that contain the - literal string. - """ - r = set() - search_terms = [] - for item in argument: - search_terms.append(item[0]) - if len(r) == 0: - r = self.evaluate(item) - else: - r = r.intersection(self.evaluate(item)) - return self.GetQuotes(' '.join(search_terms), r) - - def evaluateWord(self, argument): - return self.GetWord(argument[0]) - - def evaluateWordWildcard(self, argument): - return self.GetWordWildcard(argument[0]) - - def evaluate(self, argument): - return self._methods[argument.getName()](argument) - - def Parse(self, query): - #print(self._parser(query)[0]) - return self.evaluate(self._parser(query)[0]) - - def GetWord(self, word): - return set() - - def GetWordWildcard(self, word): - return set() - - def GetQuotes(self, search_string, tmp_result): - return set() - - def GetNot(self, not_set): - return set().difference(not_set) - - -class ParserTest(SearchQueryParser): - """Tests the parser with some search queries - tests containts a dictionary with tests and expected results. - """ - tests = { - 'help': set([1, 2, 4, 5]), - 'help or hulp': set([1, 2, 3, 4, 5]), - 'help and hulp': set([2]), - 'help hulp': set([2]), - 'help and hulp or hilp': set([2, 3, 4]), - 'help or hulp and hilp': set([1, 2, 3, 4, 5]), - 'help or hulp or hilp or halp': set([1, 2, 3, 4, 5, 6]), - '(help or hulp) and (hilp or halp)': set([3, 4, 5]), - 'help and (hilp or halp)': set([4, 5]), - '(help and (hilp or halp)) or hulp': set([2, 3, 4, 5]), - 'not help': set([3, 6, 7, 8]), - 'not hulp and halp': set([5, 6]), - 'not (help and halp)': set([1, 2, 3, 4, 6, 7, 8]), - '"help me please"': set([2]), - '"help me please" or hulp': set([2, 3]), - '"help me please" or (hulp and halp)': set([2]), - 'help*': set([1, 2, 4, 5, 8]), - 'help or hulp*': set([1, 2, 3, 4, 5]), - 'help* and hulp': set([2]), - 'help and hulp* or hilp': set([2, 3, 4]), - 'help* or hulp or hilp or halp': set([1, 2, 3, 4, 5, 6, 8]), - '(help or hulp*) and (hilp* or halp)': set([3, 4, 5]), - 'help* and (hilp* or halp*)': set([4, 5]), - '(help and (hilp* or halp)) or hulp*': set([2, 3, 4, 5]), - 'not help* and halp': set([6]), - 'not (help* and helpe*)': set([1, 2, 3, 4, 5, 6, 7]), - '"help* me please"': set([2]), - '"help* me* please" or hulp*': set([2, 3]), - '"help me please*" or (hulp and halp)': set([2]), - '"help me please" not (hulp and halp)': set([2]), - '"help me please" hulp': set([2]), - '\'help me please\' hulp': set([2]), - 'help and hilp and not holp': set([4]), - 'help hilp not holp': set([4]), - 'help hilp and not holp': set([4]), - } - - docs = { - 1: 'help', - 2: 'help me please hulp', - 3: 'hulp hilp', - 4: 'help hilp', - 5: 'halp thinks he needs help', - 6: 'he needs halp', - 7: 'nothing', - 8: 'helper', - } - - index = { - 'help': set((1, 2, 4, 5)), - 'me': set((2,)), - 'please': set((2,)), - 'hulp': set((2, 3,)), - 'hilp': set((3, 4,)), - 'halp': set((5, 6,)), - 'thinks': set((5,)), - 'he': set((5, 6,)), - 'needs': set((5, 6,)), - 'nothing': set((7,)), - 'helper': set((8,)), - } - - def GetWord(self, word): - if (self.index.has_key(word)): - return self.index[word] - else: - return set() - - def GetWordWildcard(self, word): - result = set() - for item in self.index.keys(): - if word == item[0:len(word)]: - result = result.union(self.index[item]) - return result - - def GetQuotes(self, search_string, tmp_result): - result = set() - for item in tmp_result: - if self.docs[item].count(search_string): - result.add(item) - return result - - def GetNot(self, not_set): - all = set(self.docs.keys()) - return all.difference(not_set) - - def Test(self): - all_ok = True - for item in self.tests.keys(): - print(item) - r = self.Parse(item) - e = self.tests[item] - print('Result: %s' % r) - print('Expect: %s' % e) - if e == r: - print('Test OK') - else: - all_ok = False - print('>>>>>>>>>>>>>>>>>>>>>>Test ERROR<<<<<<<<<<<<<<<<<<<<<') - print('') - return all_ok - -if __name__=='__main__': - if ParserTest().Test(): - print('All tests OK') - else: - print('One or more tests FAILED') diff --git a/libbmc/tearpages.py b/libbmc/tearpages.py deleted file mode 100644 index c21989f..0000000 --- a/libbmc/tearpages.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Author: Francois Boulogne - -import shutil -import tempfile -from PyPDF2 import PdfFileWriter, PdfFileReader -from PyPDF2.utils import PdfReadError - - -def _fixPdf(pdfFile, destination): - """ - Fix malformed pdf files when data are present after '%%EOF' - - :param pdfFile: PDF filepath - :param destination: destination - """ - tmp = tempfile.NamedTemporaryFile() - output = open(tmp.name, 'wb') - with open(pdfFile, "rb") as fh: - with open(pdfFile, "rb") as fh: - for line in fh: - output.write(line) - if b'%%EOF' in line: - break - output.close() - shutil.copy(tmp.name, destination) - - -def tearpage(filename, startpage=1): - """ - Copy filename to a tempfile, write pages startpage..N to filename. - - :param filename: PDF filepath - :param startpage: page number for the new first page - """ - # Copy the pdf to a tmp file - tmp = tempfile.NamedTemporaryFile() - shutil.copy(filename, tmp.name) - - # Read the copied pdf - try: - input_file = PdfFileReader(open(tmp.name, 'rb')) - except PdfReadError: - _fixPdf(filename, tmp.name) - input_file = PdfFileReader(open(tmp.name, 'rb')) - # Seek for the number of pages - num_pages = input_file.getNumPages() - - # Write pages excepted the first one - output_file = PdfFileWriter() - for i in range(startpage, num_pages): - output_file.addPage(input_file.getPage(i)) - - tmp.close() - outputStream = open(filename, "wb") - output_file.write(outputStream) diff --git a/libbmc/tests/src/arxiv.bib b/libbmc/tests/src/arxiv.bib deleted file mode 100644 index 073ab5a..0000000 --- a/libbmc/tests/src/arxiv.bib +++ /dev/null @@ -1,21 +0,0 @@ -@article{1303.3130v1, - abstract={We study the role of the dipolar interaction, correctly accounting for the -Dipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold -bosons. We first show how the DIR affects the lowest-energy states of two -particles in a harmonic trap. Then, we consider a deep optical lattice loaded -with ultracold dipolar bosons. We describe this many-body system using an -atom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the -phase diagram at T=0 by exact diagonalization of a small-sized system. In -particular, the resonance strongly modifies the range of parameters for which a -mass density wave should occur.}, - archiveprefix={arXiv}, - author={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati}, - eprint={1303.3130v1}, - link={http://arxiv.org/abs/1303.3130v1}, - month={Mar}, - primaryclass={cond-mat.quant-gas}, - title={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical -Lattice}, - year={2013}, -} - diff --git a/libbmc/tests/src/doi.bib b/libbmc/tests/src/doi.bib deleted file mode 100644 index 77a9350..0000000 --- a/libbmc/tests/src/doi.bib +++ /dev/null @@ -1,12 +0,0 @@ -@article{Hou_2013, - doi = {10.1103/physreva.88.043630}, - url = {http://dx.doi.org/10.1103/physreva.88.043630}, - year = 2013, - month = {oct}, - publisher = {American Physical Society ({APS})}, - volume = {88}, - number = {4}, - author = {Yan-Hua Hou and Lev P. Pitaevskii and Sandro Stringari}, - title = {First and second sound in a highly elongated Fermi gas at unitarity}, - journal = {Phys. Rev. A} -} \ No newline at end of file diff --git a/libbmc/tests/src/isbn.bib b/libbmc/tests/src/isbn.bib deleted file mode 100644 index e85441f..0000000 --- a/libbmc/tests/src/isbn.bib +++ /dev/null @@ -1,7 +0,0 @@ -@book{9780198507192, - title = {Bose-Einstein Condensation}, - author = {Lev. P. Pitaevskii and S. Stringari}, - isbn = {9780198507192}, - year = {2004}, - publisher = {Clarendon Press} -} \ No newline at end of file diff --git a/libbmc/tests/src/test.djvu b/libbmc/tests/src/test.djvu deleted file mode 100644 index 7d30dcc..0000000 Binary files a/libbmc/tests/src/test.djvu and /dev/null differ diff --git a/libbmc/tests/src/test.pdf b/libbmc/tests/src/test.pdf deleted file mode 100644 index 1493948..0000000 Binary files a/libbmc/tests/src/test.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_arxiv_doi_conflict.pdf b/libbmc/tests/src/test_arxiv_doi_conflict.pdf deleted file mode 100644 index 9bf310d..0000000 Binary files a/libbmc/tests/src/test_arxiv_doi_conflict.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_arxiv_multi.pdf b/libbmc/tests/src/test_arxiv_multi.pdf deleted file mode 100644 index 33fcd9a..0000000 Binary files a/libbmc/tests/src/test_arxiv_multi.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_arxiv_published.pdf b/libbmc/tests/src/test_arxiv_published.pdf deleted file mode 100644 index d27250d..0000000 Binary files a/libbmc/tests/src/test_arxiv_published.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_book.djvu b/libbmc/tests/src/test_book.djvu deleted file mode 100644 index 94b9465..0000000 Binary files a/libbmc/tests/src/test_book.djvu and /dev/null differ diff --git a/libbmc/tests/src/test_book.pdf b/libbmc/tests/src/test_book.pdf deleted file mode 100644 index b723280..0000000 Binary files a/libbmc/tests/src/test_book.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_hal.pdf b/libbmc/tests/src/test_hal.pdf deleted file mode 100644 index 65289f7..0000000 Binary files a/libbmc/tests/src/test_hal.pdf and /dev/null differ diff --git a/libbmc/tests/src/test_watermark.pdf b/libbmc/tests/src/test_watermark.pdf deleted file mode 100644 index 5d7bfd1..0000000 Binary files a/libbmc/tests/src/test_watermark.pdf and /dev/null differ diff --git a/libbmc/tests/test_backend.py b/libbmc/tests/test_backend.py deleted file mode 100644 index e81b77f..0000000 --- a/libbmc/tests/test_backend.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- -from __future__ import unicode_literals -import unittest -from libbmc.backend import * -import bibtexparser -import os -import shutil -import tempfile - - -class TestFetcher(unittest.TestCase): - def setUp(self): - config.set("folder", tempfile.mkdtemp()+"/") - self.bibtex_article_string = """ -@article{1303.3130v1, - abstract={We study the role of the dipolar interaction, correctly accounting for the -Dipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold -bosons. We first show how the DIR affects the lowest-energy states of two -particles in a harmonic trap. Then, we consider a deep optical lattice loaded -with ultracold dipolar bosons. We describe this many-body system using an -atom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the -phase diagram at T=0 by exact diagonalization of a small-sized system. In -particular, the resonance strongly modifies the range of parameters for which a -mass density wave should occur.}, - archiveprefix={arXiv}, - author={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati}, - eprint={1303.3130v1}, - file={%sN_Bartolo_A_Recati-j-2013.pdf}, - link={http://arxiv.org/abs/1303.3130v1}, - month={Mar}, - primaryclass={cond-mat.quant-gas}, - tag={}, - title={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical -Lattice}, - year={2013}, -}""" % config.get("folder") - self.bibtex_article = bibtexparser.loads(self.bibtex_article_string).entries_dict - self.bibtex_article = self.bibtex_article[list(self.bibtex_article.keys())[0]] - - self.bibtex_book_string = """ -@book{9780521846516, - author={C. J. Pethick and H. Smith}, - isbn={9780521846516}, - publisher={Cambridge University Press}, - title={Bose-Einstein Condensation In Dilute Gases}, - year={2008}, -} -""" - self.bibtex_book = bibtexparser.loads(self.bibtex_book_string).entries_dict - self.bibtex_book = self.bibtex_book[list(self.bibtex_book.keys())[0]] - - def test_getNewName_article(self): - self.assertEqual(getNewName("test.pdf", self.bibtex_article), - config.get("folder")+"N_Bartolo_A_Recati-j-2013-v1.pdf") - - def test_getNewName_article_override(self): - self.assertEqual(getNewName("test.pdf", self.bibtex_article, override_format="%f"), - config.get("folder")+"N_Bartolo.pdf") - - def test_getNewName_book(self): - self.assertEqual(getNewName("test.pdf", self.bibtex_book), - config.get("folder")+"C_J_Pethick_H_Smith-Bose-Einstein_Condensation_In_Dilute_Gases.pdf") - - def test_getNewName_book_override(self): - self.assertEqual(getNewName("test.pdf", self.bibtex_book, override_format="%a"), - config.get("folder")+"C_J_Pethick_H_Smith.pdf") - - def test_bibtexAppend(self): - bibtexAppend(self.bibtex_article) - with open(config.get("folder")+'index.bib', 'r') as fh: - self.assertEqual(fh.read(), - '@article{1303.3130v1,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={'+config.get("folder")+'N_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n') - - def test_bibtexEdit(self): - bibtexAppend(self.bibtex_article) - bibtexEdit(self.bibtex_article['ID'], {'ID': 'bidule'}) - with open(config.get("folder")+'index.bib', 'r') as fh: - self.assertEqual(fh.read(), - '@article{bidule,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={'+config.get("folder")+'N_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n') - - def test_bibtexRewrite(self): - bibtexAppend(self.bibtex_book) - bibtexRewrite({0: self.bibtex_article}) - with open(config.get("folder")+'index.bib', 'r') as fh: - self.assertEqual(fh.read(), - '@article{1303.3130v1,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={%sN_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n' % config.get("folder")) - - def test_deleteId(self): - self.bibtex_article['file'] = config.get("folder")+'test.pdf' - bibtexAppend(self.bibtex_article) - open(config.get("folder")+'test.pdf', 'w').close() - deleteId(self.bibtex_article['ID']) - with open(config.get("folder")+'index.bib', 'r') as fh: - self.assertEqual(fh.read().strip(), "") - self.assertFalse(os.path.isfile(config.get("folder")+'test.pdf')) - - def test_deleteFile(self): - self.bibtex_article['file'] = config.get("folder")+'test.pdf' - bibtexAppend(self.bibtex_article) - open(config.get("folder")+'test.pdf', 'w').close() - deleteFile(self.bibtex_article['file']) - with open(config.get("folder")+'index.bib', 'r') as fh: - self.assertEqual(fh.read().strip(), "") - self.assertFalse(os.path.isfile(config.get("folder")+'test.pdf')) - - def test_diffFilesIndex(self): - # TODO - return - - def test_getBibtex(self): - bibtexAppend(self.bibtex_article) - got = getBibtex(self.bibtex_article['ID']) - self.assertEqual(got, self.bibtex_article) - - def test_getBibtex_id(self): - bibtexAppend(self.bibtex_article) - got = getBibtex(self.bibtex_article['ID'], file_id='id') - self.assertEqual(got, self.bibtex_article) - - def test_getBibtex_file(self): - self.bibtex_article['file'] = config.get("folder")+'test.pdf' - open(config.get("folder")+'test.pdf', 'w').close() - bibtexAppend(self.bibtex_article) - got = getBibtex(self.bibtex_article['file'], file_id='file') - self.assertEqual(got, self.bibtex_article) - - def test_getBibtex_clean(self): - config.set("ignore_fields", ['ID', 'abstract']) - bibtexAppend(self.bibtex_article) - got = getBibtex(self.bibtex_article['ID'], clean=True) - for i in config.get("ignore_fields"): - self.assertNotIn(i, got) - - def test_getEntries(self): - bibtexAppend(self.bibtex_article) - self.assertEqual(getEntries(), - [self.bibtex_article['ID']]) - - def test_updateArxiv(self): - # TODO - return - - def test_search(self): - # TODO - return - - def tearDown(self): - shutil.rmtree(config.get("folder")) - -if __name__ == '__main__': - unittest.main() diff --git a/libbmc/tests/test_config.py b/libbmc/tests/test_config.py deleted file mode 100644 index b14386c..0000000 --- a/libbmc/tests/test_config.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- -from __future__ import unicode_literals -import unittest -import json -import os -import tempfile -import shutil -from libbmc.config import Config - - -class TestConfig(unittest.TestCase): - def setUp(self): - self.folder = tempfile.mkdtemp()+"/" - self.default_config = {"folder": os.path.expanduser("~/Papers/"), - "proxies": [''], - "format_articles": "%f_%l-%j-%Y%v", - "format_books": "%a-%t", - "format_custom": [], - "ignore_fields": ["file", "doi", "tag"]} - - def tearDown(self): - shutil.rmtree(self.folder) - - def test_load_without_file(self): - config = Config(base_config_path=self.folder) - self.assertEqual(config.as_dict(), self.default_config) - with open(self.folder+"bmc.json", 'r') as fh: - read = json.loads(fh.read()) - self.assertEqual(read, self.default_config) - - def test_load_with_file(self): - config = self.default_config - config["foo"] = "bar" - with open(self.folder+"bmc.json", 'w') as fh: - json.dump(config, fh) - config_read = Config(base_config_path=self.folder) - self.assertEqual(config, config_read.as_dict()) - - def test_get(self): - config = Config(base_config_path=self.folder) - self.assertEqual(config.get("proxies"), ['']) - - def test_set(self): - config = Config(base_config_path=self.folder) - config.set("foo", "bar") - self.assertEqual(config.get("foo"), "bar") - - def test_save(self): - config = Config(base_config_path=self.folder) - config.set("foo", "bar") - config.save() - with open(self.folder+"bmc.json", 'r') as fh: - read = json.loads(fh.read()) - self.assertEqual(read, config.as_dict()) - - def test_masks(self): - with open(self.folder+"masks.py", 'w') as fh: - fh.write("def f(x): return x") - config = Config(base_config_path=self.folder) - self.assertEqual("foo", config.get("format_custom")[0]("foo")) - - -if __name__ == '__main__': - unittest.main() diff --git a/libbmc/tests/test_fetcher.py b/libbmc/tests/test_fetcher.py deleted file mode 100644 index 3fbf9fb..0000000 --- a/libbmc/tests/test_fetcher.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- - -import unittest -from libbmc.fetcher import * - - -class TestFetcher(unittest.TestCase): - def setUp(self): - with open("libbmc/tests/src/doi.bib", 'r') as fh: - self.doi_bib = fh.read() - with open("libbmc/tests/src/arxiv.bib", 'r') as fh: - self.arxiv_bib = fh.read() - with open("libbmc/tests/src/isbn.bib", 'r') as fh: - self.isbn_bib = fh.read() - - def test_download(self): - dl, contenttype = download('http://arxiv.org/pdf/1312.4006.pdf') - self.assertIn(contenttype, ['pdf', 'djvu']) - self.assertNotEqual(dl, '') - - def test_download_invalid_type(self): - self.assertFalse(download('http://phyks.me/')[0]) - - def test_download_invalid_url(self): - self.assertFalse(download('a')[0]) - - def test_findISBN_DJVU(self): - # ISBN is incomplete in this test because my djvu file is bad - self.assertEqual(findISBN("libbmc/tests/src/test_book.djvu"), '978295391873') - - def test_findISBN_PDF(self): - self.assertEqual(findISBN("libbmc/tests/src/test_book.pdf"), '9782953918731') - - def test_findISBN_False(self): - self.assertFalse(findISBN("libbmc/tests/src/test.pdf")) - - def test_isbn2Bib(self): - self.assertEqual(isbn2Bib('0198507194'), self.isbn_bib) - - def test_isbn2Bib_False(self): - self.assertEqual(isbn2Bib('foo'), '') - - def test_findDOI_PDF(self): - self.assertEqual(findArticleID("libbmc/tests/src/test.pdf"), - ("DOI", "10.1103/physrevlett.112.253201")) - - def test_findOnlyDOI(self): - self.assertEqual(findArticleID("libbmc/tests/src/test.pdf", - only=["DOI"]), - ("DOI", "10.1103/physrevlett.112.253201")) - - def test_findDOID_DJVU(self): - # DOI is incomplete in this test because my djvu file is bad - self.assertEqual(findArticleID("libbmc/tests/src/test.djvu"), - ("DOI", "10.1103/physrevlett.112")) - - def test_findDOI_False(self): - self.assertFalse(findArticleID("libbmc/tests/src/test_arxiv_multi.pdf", - only=["DOI"])[0]) - - def test_doi2Bib(self): - self.assertEqual(doi2Bib('10.1103/physreva.88.043630'), self.doi_bib) - - def test_doi2Bib_False(self): - self.assertEqual(doi2Bib('blabla'), '') - - def test_findArXivId(self): - self.assertEqual(findArticleID("libbmc/tests/src/test_arxiv_multi.pdf"), - ("arXiv", '1303.3130v1')) - - def test_findOnlyArXivId(self): - self.assertEqual(findArticleID("libbmc/tests/src/test_arxiv_multi.pdf", - only=["arXiv"]), - ("arXiv", '1303.3130v1')) - - def test_findArticleID(self): - # cf https://github.com/Phyks/BMC/issues/19 - self.assertEqual(findArticleID("libbmc/tests/src/test_arxiv_doi_conflict.pdf"), - ("arXiv", '1107.4487v1')) - - def test_arXiv2Bib(self): - self.assertEqual(arXiv2Bib('1303.3130v1'), self.arxiv_bib) - - def test_arXiv2Bib_False(self): - self.assertEqual(arXiv2Bib('blabla'), '') - - def test_findHALId(self): - self.assertTupleEqual(findHALId("libbmc/tests/src/test_hal.pdf"), - ('hal-00750893', '3')) - -if __name__ == '__main__': - unittest.main() diff --git a/libbmc/tests/test_tools.py b/libbmc/tests/test_tools.py deleted file mode 100644 index 5656e90..0000000 --- a/libbmc/tests/test_tools.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- -from __future__ import unicode_literals - -import unittest -from libbmc.tools import * - - -class TestTools(unittest.TestCase): - def test_slugify(self): - self.assertEqual(slugify(u"à&é_truc.pdf"), "ae_trucpdf") - - def test_parsed2Bibtex(self): - parsed = {'ENTRYTYPE': 'article', 'ID': 'test', 'field1': 'test1', - 'field2': 'test2'} - expected = ('@article{test,\n\tfield1={test1},\n' + - '\tfield2={test2},\n}\n\n') - self.assertEqual(parsed2Bibtex(parsed), expected) - - def test_getExtension(self): - self.assertEqual(getExtension('test.ext'), '.ext') - - def test_replaceAll(self): - replace_dict = {"test": "bidule", "machin": "chose"} - self.assertEqual(replaceAll("test machin truc", replace_dict), - "bidule chose truc") - -if __name__ == '__main__': - unittest.main() diff --git a/libbmc/tools.py b/libbmc/tools.py deleted file mode 100644 index dc29f3b..0000000 --- a/libbmc/tools.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf8 -*- -# ----------------------------------------------------------------------------- -# "THE NO-ALCOHOL BEER-WARE LICENSE" (Revision 42): -# Phyks (webmaster@phyks.me) wrote this file. As long as you retain this notice -# you can do whatever you want with this stuff (and you can also do whatever -# you want with this stuff without retaining it, but that's not cool...). If we -# meet some day, and you think this stuff is worth it, you can buy me a -# beer soda in return. -# Phyks -# ----------------------------------------------------------------------------- - - -from __future__ import print_function, unicode_literals -import os -import re -import sys -if os.name == "posix": - from termios import tcflush, TCIOFLUSH - -try: - input = raw_input -except NameError: - pass - -_slugify_strip_re = re.compile(r'[^\w\s-]') -_slugify_hyphenate_re = re.compile(r'[\s]+') - - -def slugify(value): - """Normalizes string, converts to lowercase, removes non-alpha characters, - and converts spaces to hyphens to have nice filenames. - - From Django's "django/template/defaultfilters.py". - """ - import unicodedata - try: - unicode_type = unicode - except NameError: - unicode_type = str - if not isinstance(value, unicode_type): - value = unicode_type(value) - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') - value = unicode_type(_slugify_strip_re.sub('', value).strip()) - return _slugify_hyphenate_re.sub('_', value) - - -def parsed2Bibtex(parsed): - """Convert a single bibtex entry dict to bibtex string""" - bibtex = '@'+parsed['ENTRYTYPE']+'{'+parsed['ID']+",\n" - - for field in [i for i in sorted(parsed) if i not in ['ENTRYTYPE', 'ID']]: - bibtex += "\t"+field+"={"+parsed[field]+"},\n" - bibtex += "}\n\n" - return bibtex - - -def getExtension(filename): - """Get the extension of filename""" - return filename[filename.rfind('.'):] - - -def replaceAll(text, dic): - """Replace all the dic keys by the associated item in text""" - for i, j in dic.items(): - text = text.replace(i, j) - return text - - -def rawInput(string): - """Flush stdin and then prompt the user for something""" - if os.name == "posix": - tcflush(sys.stdin, TCIOFLUSH) - return input(string) - - -def warning(*objs): - """Write warnings to stderr""" - print("WARNING: ", *objs, file=sys.stderr) - - -def listDir(path): - """List all files in path directory, works recursively - - Return files list - """ - filenames = [] - for root, dirs, files in os.walk(path): - for i in files: - filenames.append(os.path.join(root, i)) - return filenames diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b155715 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +libbmc diff --git a/setup.py b/setup.py index 37be75a..592d201 100644 --- a/setup.py +++ b/setup.py @@ -4,12 +4,13 @@ from distutils.core import setup setup( name = 'BMC', - version = "0.3dev", + version = "0.4", url = "https://github.com/Phyks/BMC", - author = "", - license = "no-alcohol beer-ware license", - author_email = "", - description = "simple script to download and store your articles", + author = "Phyks (Lucas Verney)", + license = "MIT License", + author_email = "phyks@phyks.me", + description = "Simple script to download and store your articles", + # TODO packages = ['libbmc'], scripts = ['bmc.py'], )