bmc/bmc.py

680 lines
26 KiB
Python
Executable File

#!/usr/bin/env python
# -*- coding: utf8 -*-
from __future__ import unicode_literals
import argparse
import os
import shutil
import subprocess
import sys
import tempfile
import bibtexparser
from codecs import open
from libbmc.config import Config
from libbmc import backend
from libbmc import fetcher
from libbmc import tearpages
from libbmc import tools
config = Config()
EDITOR = os.environ.get('EDITOR') if os.environ.get('EDITOR') else 'vim'
def checkBibtex(filename, bibtex_string):
print("The bibtex entry found for "+filename+" is:")
bibtex = bibtexparser.loads(bibtex_string)
bibtex = bibtex.entries_dict
try:
bibtex = bibtex[list(bibtex.keys())[0]]
# Check entries are correct
if "title" not in bibtex:
raise AssertionError
if "authors" not in bibtex and "author" not in bibtex:
raise AssertionError
if "year" not in bibtex:
raise AssertionError
# Print the bibtex and confirm
print(tools.parsed2Bibtex(bibtex))
check = tools.rawInput("Is it correct? [Y/n] ")
except KeyboardInterrupt:
sys.exit()
except (IndexError, KeyError, AssertionError):
print("Missing author, year or title in bibtex.")
check = 'n'
try:
old_filename = bibtex['file']
except KeyError:
old_filename = False
while check.lower() == 'n':
with tempfile.NamedTemporaryFile(suffix=".tmp") as tmpfile:
tmpfile.write(bibtex_string.encode('utf-8'))
tmpfile.flush()
subprocess.call([EDITOR, tmpfile.name])
tmpfile.seek(0)
bibtex = bibtexparser.loads(tmpfile.read().decode('utf-8')+"\n")
bibtex = bibtex.entries_dict
try:
bibtex = bibtex[list(bibtex.keys())[0]]
except (IndexError, KeyError):
tools.warning("Invalid bibtex entry")
bibtex_string = ''
tools.rawInput("Press Enter to go back to editor.")
continue
if('authors' not in bibtex and 'title' not in bibtex and 'year' not in
bibtex):
tools.warning("Invalid bibtex entry")
bibtex_string = ''
tools.rawInput("Press Enter to go back to editor.")
continue
if old_filename is not False and 'file' not in bibtex:
tools.warning("Invalid bibtex entry. No filename given.")
tools.rawInput("Press Enter to go back to editor.")
check = 'n'
else:
bibtex_string = tools.parsed2Bibtex(bibtex)
print("\nThe bibtex entry for "+filename+" is:")
print(bibtex_string)
check = tools.rawInput("Is it correct? [Y/n] ")
if old_filename is not False and old_filename != bibtex['file']:
try:
print("Moving file to new location…")
shutil.move(old_filename, bibtex['file'])
except shutil.Error:
tools.warning("Unable to move file "+old_filename+" to " +
bibtex['file']+". You should check it manually.")
return bibtex
def addFile(src, filetype, manual, autoconfirm, tag, rename=True):
"""
Add a file to the library
"""
doi = False
arxiv = False
isbn = False
if not manual:
try:
if filetype == 'article' or filetype is None:
id_type, article_id = fetcher.findArticleID(src)
if id_type == "DOI":
doi = article_id
elif id_type == "arXiv":
arxiv = article_id
if filetype == 'book' or (doi is False and arxiv is False and
filetype is None):
isbn = fetcher.findISBN(src)
except KeyboardInterrupt:
doi = False
arxiv = False
isbn = False
if doi is False and isbn is False and arxiv is False:
if filetype is None:
tools.warning("Could not determine the DOI nor the arXiv id nor " +
"the ISBN for "+src+". Switching to manual entry.")
doi_arxiv_isbn = ''
while(doi_arxiv_isbn not in
['doi', 'arxiv', 'isbn', 'manual', 'skip']):
doi_arxiv_isbn = (tools.rawInput("DOI / arXiv " +
"/ ISBN / manual / skip? ").
lower())
if doi_arxiv_isbn == 'doi':
doi = tools.rawInput('DOI? ')
elif doi_arxiv_isbn == 'arxiv':
arxiv = tools.rawInput('arXiv id? ')
elif doi_arxiv_isbn == 'isbn':
isbn = tools.rawInput('ISBN? ')
elif doi_arxiv_isbn == 'skip':
return False
elif filetype == 'article':
tools.warning("Could not determine the DOI nor the arXiv id for " +
src+", switching to manual entry.")
doi_arxiv = ''
while doi_arxiv not in ['doi', 'arxiv', 'manual', 'skip']:
doi_arxiv = (tools.rawInput("DOI / arXiv / manual / skip? ").
lower())
if doi_arxiv == 'doi':
doi = tools.rawInput('DOI? ')
elif doi_arxiv == 'arxiv':
arxiv = tools.rawInput('arXiv id? ')
elif doi_arxiv == 'skip':
return False
elif filetype == 'book':
isbn_manual = ''
while isbn_manual not in ['isbn', 'manual', 'skip']:
isbn_manual = tools.rawInput("ISBN / manual / skip? ").lower()
if isbn_manual == 'isbn':
isbn = (tools.rawInput('ISBN? ').
replace(' ', '').
replace('-', ''))
elif isbn_manual == 'skip':
return False
elif doi is not False:
print("DOI for "+src+" is "+doi+".")
elif arxiv is not False:
print("ArXiv id for "+src+" is "+arxiv+".")
elif isbn is not False:
print("ISBN for "+src+" is "+isbn+".")
if doi is not False and doi != '':
# Add extra \n for bibtexparser
bibtex = fetcher.doi2Bib(doi).strip().replace(',', ",\n")+"\n"
elif arxiv is not False and arxiv != '':
bibtex = fetcher.arXiv2Bib(arxiv).strip().replace(',', ",\n")+"\n"
elif isbn is not False and isbn != '':
# Idem
bibtex = fetcher.isbn2Bib(isbn).strip()+"\n"
else:
bibtex = ''
bibtex = bibtexparser.loads(bibtex)
bibtex = bibtex.entries_dict
if len(bibtex) > 0:
bibtex_name = list(bibtex.keys())[0]
bibtex = bibtex[bibtex_name]
bibtex_string = tools.parsed2Bibtex(bibtex)
else:
bibtex_string = ''
if not autoconfirm:
bibtex = checkBibtex(src, bibtex_string)
if not autoconfirm:
tag = tools.rawInput("Tag for this paper (leave empty for default) ? ")
else:
tag = args.tag
bibtex['tag'] = tag
if rename:
new_name = backend.getNewName(src, bibtex, tag)
while os.path.exists(new_name):
tools.warning("file "+new_name+" already exists.")
default_rename = new_name.replace(tools.getExtension(new_name),
" (2)" +
tools.getExtension(new_name))
rename = tools.rawInput("New name ["+default_rename+"]? ")
if rename == '':
new_name = default_rename
else:
new_name = rename
try:
shutil.copy2(src, new_name)
except shutil.Error:
new_name = False
sys.exit("Unable to move file to library dir " +
config.get("folder")+".")
else:
new_name = src
bibtex['file'] = os.path.abspath(new_name)
# Remove first page of IOP papers
try:
if 'IOP' in bibtex['publisher'] and bibtex['ENTRYTYPE'] == 'article':
tearpages.tearpage(new_name)
except (KeyError, shutil.Error, IOError):
pass
backend.bibtexAppend(bibtex)
return new_name
def editEntry(entry, file_id='both'):
bibtex = backend.getBibtex(entry, file_id)
if bibtex is False:
tools.warning("Entry "+entry+" does not exist.")
return False
if file_id == 'file':
filename = entry
else:
filename = bibtex['file']
new_bibtex = checkBibtex(filename, tools.parsed2Bibtex(bibtex))
# Tag update
if new_bibtex['tag'] != bibtex['tag']:
print("Editing tag, moving file.")
new_name = backend.getNewName(new_bibtex['file'],
new_bibtex,
new_bibtex['tag'])
while os.path.exists(new_name):
tools.warning("file "+new_name+" already exists.")
default_rename = new_name.replace(tools.getExtension(new_name),
" (2)" +
tools.getExtension(new_name))
rename = tools.rawInput("New name ["+default_rename+"]? ")
if rename == '':
new_name = default_rename
else:
new_name = rename
new_bibtex['file'] = new_name
try:
shutil.move(bibtex['file'], new_bibtex['file'])
except shutil.Error:
tools.warning('Unable to move file '+bibtex['file']+' to ' +
new_bibtex['file'] + ' according to tag edit.')
try:
if not os.listdir(os.path.dirname(bibtex['file'])):
os.rmdir(os.path.dirname(bibtex['file']))
except OSError:
tools.warning("Unable to delete empty tag dir " +
os.path.dirname(bibtex['file']))
try:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
index = bibtexparser.load(fh)
index = index.entries_dict
except (TypeError, IOError):
tools.warning("Unable to open index file.")
return False
index[new_bibtex['ID']] = new_bibtex
backend.bibtexRewrite(index)
return True
def downloadFile(url, filetype, manual, autoconfirm, tag):
print('Downloading '+url)
dl, contenttype = fetcher.download(url)
if dl is not False:
print('Download finished')
tmp = tempfile.NamedTemporaryFile(suffix='.'+contenttype)
with open(tmp.name, 'wb+') as fh:
fh.write(dl)
new_name = addFile(tmp.name, filetype, manual, autoconfirm, tag)
if new_name is False:
return False
tmp.close()
return new_name
else:
tools.warning("Could not fetch "+url)
return False
def openFile(ident):
try:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = bibtexparser.load(fh)
bibtex = bibtex.entries_dict
except (TypeError, IOError):
tools.warning("Unable to open index file.")
return False
if ident not in list(bibtex.keys()):
return False
else:
subprocess.Popen(['xdg-open', bibtex[ident]['file']])
return True
def resync():
diff = backend.diffFilesIndex()
if diff is False:
return False
for key in diff:
entry = diff[key]
if entry['file'] == '':
print("\nFound entry in index without associated file: " +
entry['ID'])
print("Title:\t"+entry['title'])
loop = True
while confirm:
filename = tools.rawInput("File to import for this entry " +
"(leave empty to delete the " +
"entry)? ")
if filename == '':
break
else:
if 'doi' in list(entry.keys()):
doi = fetcher.findArticleID(filename, only=["DOI"])
if doi is not False and doi != entry['doi']:
loop = tools.rawInput("Found DOI does not " +
"match bibtex entry " +
"DOI, continue anyway " +
"? [y/N]")
loop = (loop.lower() != 'y')
if 'Eprint' in list(entry.keys()):
arxiv = fetcher.findArticleID(filename, only=["arXiv"])
if arxiv is not False and arxiv != entry['Eprint']:
loop = tools.rawInput("Found arXiv id does " +
"not match bibtex " +
"entry arxiv id, " +
"continue anyway ? [y/N]")
loop = (loop.lower() != 'y')
if 'isbn' in list(entry.keys()):
isbn = fetcher.findISBN(filename)
if isbn is not False and isbn != entry['isbn']:
loop = tools.rawInput("Found ISBN does not " +
"match bibtex entry " +
"ISBN, continue anyway " +
"? [y/N]")
loop = (loop.lower() != 'y')
continue
if filename == '':
backend.deleteId(entry['ID'])
print("Deleted entry \""+entry['ID']+"\".")
else:
new_name = backend.getNewName(filename, entry)
try:
shutil.copy2(filename, new_name)
print("Imported new file "+filename+" for entry " +
entry['ID']+".")
except shutil.Error:
new_name = False
sys.exit("Unable to move file to library dir " +
config.get("folder")+".")
backend.bibtexEdit(entry['ID'], {'file': filename})
else:
print("Found file without any associated entry in index:")
print(entry['file'])
action = ''
while action.lower() not in ['import', 'delete']:
action = tools.rawInput("What to do? [import / delete] ")
action = action.lower()
if action == 'import':
tmp = tempfile.NamedTemporaryFile()
shutil.copy(entry['file'], tmp.name)
filetype = tools.getExtension(entry['file'])
try:
os.remove(entry['file'])
except OSError:
tools.warning("Unable to delete file "+entry['file'])
if not addFile(tmp.name, filetype):
tools.warning("Unable to reimport file "+entry['file'])
tmp.close()
else:
backend.deleteFile(entry['file'])
print(entry['file'] + " removed from disk and " +
"index.")
# Check for empty tag dirs
for i in os.listdir(config.get("folder")):
if os.path.isdir(i) and not os.listdir(config.get("folder") + i):
try:
os.rmdir(config.get("folder") + i)
except OSError:
tools.warning("Found empty tag dir "+config.get("folder") + i +
" but could not delete it.")
def update(entry):
update = backend.updateArXiv(entry)
if update is not False:
print("New version found for "+entry)
print("\t Title: "+update['title'])
confirm = tools.rawInput("Download it ? [Y/n] ")
if confirm.lower() == 'n':
return
new_name = downloadFile('http://arxiv.org/pdf/'+update['eprint'],
'article', False)
if new_name is not False:
print(update['eprint']+" successfully imported as "+new_name)
else:
tools.warning("An error occurred while downloading "+url)
confirm = tools.rawInput("Delete previous version ? [y/N] ")
if confirm.lower() == 'y':
if not backend.deleteId(entry):
if not backend.deleteFile(entry):
tools.warning("Unable to remove previous version.")
return
print("Previous version successfully deleted.")
def commandline_arg(bytestring):
# UTF-8 encoding for python2
if sys.version_info >= (3, 0):
unicode_string = bytestring
else:
unicode_string = bytestring.decode(sys.getfilesystemencoding())
return unicode_string
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="A bibliography " +
"management tool.")
subparsers = parser.add_subparsers(help="sub-command help", dest='parser')
subparsers.required = True # Fix for Python 3.3.5
parser_download = subparsers.add_parser('download', help="download help")
parser_download.add_argument('-t', '--type', default=None,
choices=['article', 'book'],
help="type of the file to download",
type=commandline_arg)
parser_download.add_argument('-m', '--manual', default=False,
action='store_true',
help="disable auto-download of bibtex")
parser_download.add_argument('-y', default=False,
help="Confirm all")
parser_download.add_argument('--tag', default='',
help="Tag", type=commandline_arg)
parser_download.add_argument('--keep', default=False,
help="Do not remove the file")
parser_download.add_argument('url', nargs='+',
help="url of the file to import",
type=commandline_arg)
parser_download.set_defaults(func='download')
parser_import = subparsers.add_parser('import', help="import help")
parser_import.add_argument('-t', '--type', default=None,
choices=['article', 'book'],
help="type of the file to import",
type=commandline_arg)
parser_import.add_argument('-m', '--manual', default=False,
action='store_true',
help="disable auto-download of bibtex")
parser_import.add_argument('-y', default=False,
help="Confirm all")
parser_import.add_argument('--tag', default='', help="Tag",
type=commandline_arg)
parser_import.add_argument('--in-place', default=False,
dest="inplace", action='store_true',
help="Leave the imported file in place",)
parser_import.add_argument('file', nargs='+',
help="path to the file to import",
type=commandline_arg)
parser_import.add_argument('--skip', nargs='+',
help="path to files to skip", default=[],
type=commandline_arg)
parser_import.set_defaults(func='import')
parser_delete = subparsers.add_parser('delete', help="delete help")
parser_delete.add_argument('entries', metavar='entry', nargs='+',
help="a filename or an identifier",
type=commandline_arg)
parser_delete.add_argument('--skip', nargs='+',
help="path to files to skip", default=[],
type=commandline_arg)
group = parser_delete.add_mutually_exclusive_group()
group.add_argument('--id', action="store_true", default=False,
help="id based deletion")
group.add_argument('--file', action="store_true", default=False,
help="file based deletion")
parser_delete.add_argument('-f', '--force', default=False,
action='store_true',
help="delete without confirmation")
parser_delete.set_defaults(func='delete')
parser_edit = subparsers.add_parser('edit', help="edit help")
parser_edit.add_argument('entries', metavar='entry', nargs='+',
help="a filename or an identifier",
type=commandline_arg)
parser_edit.add_argument('--skip', nargs='+',
help="path to files to skip", default=[],
type=commandline_arg)
group = parser_edit.add_mutually_exclusive_group()
group.add_argument('--id', action="store_true", default=False,
help="id based deletion")
group.add_argument('--file', action="store_true", default=False,
help="file based deletion")
parser_edit.set_defaults(func='edit')
parser_list = subparsers.add_parser('list', help="list help")
parser_list.set_defaults(func='list')
parser_search = subparsers.add_parser('search', help="search help")
parser_search.set_defaults(func='search')
parser_open = subparsers.add_parser('open', help="open help")
parser_open.add_argument('ids', metavar='id', nargs='+',
help="an identifier",
type=commandline_arg)
parser_open.set_defaults(func='open')
parser_export = subparsers.add_parser('export', help="export help")
parser_export.add_argument('ids', metavar='id', nargs='+',
help="an identifier",
type=commandline_arg)
parser_export.set_defaults(func='export')
parser_resync = subparsers.add_parser('resync', help="resync help")
parser_resync.set_defaults(func='resync')
parser_update = subparsers.add_parser('update', help="update help")
parser_update.add_argument('--entries', metavar='entry', nargs='+',
help="a filename or an identifier",
type=commandline_arg)
parser_update.set_defaults(func='update')
parser_search = subparsers.add_parser('search', help="search help")
parser_search.add_argument('query', metavar='entry', nargs='+',
help="your query, see README for more info.",
type=commandline_arg)
parser_search.set_defaults(func='search')
args = parser.parse_args()
try:
if args.func == 'download':
skipped = []
for url in args.url:
new_name = downloadFile(url, args.type, args.manual, args.y,
args.tag)
if new_name is not False:
print(url+" successfully imported as "+new_name)
else:
tools.warning("An error occurred while downloading "+url)
skipped.append(url)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit()
if args.func == 'import':
skipped = []
for filename in list(set(args.file) - set(args.skip)):
new_name = addFile(filename, args.type, args.manual, args.y,
args.tag, not args.inplace)
if new_name is not False:
print(filename+" successfully imported as " +
new_name+".")
else:
tools.warning("An error occurred while importing " +
filename)
skipped.append(filename)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit()
elif args.func == 'delete':
skipped = []
for filename in list(set(args.entries) - set(args.skip)):
if not args.force:
confirm = tools.rawInput("Are you sure you want to " +
"delete "+filename+" ? [y/N] ")
else:
confirm = 'y'
if confirm.lower() == 'y':
if args.file or not backend.deleteId(filename, args.keep):
if(args.id or
not backend.deleteFile(filename, args.keep)):
tools.warning("Unable to delete "+filename)
sys.exit(1)
print(filename+" successfully deleted.")
else:
skipped.append(filename)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit()
elif args.func == 'edit':
for filename in list(set(args.entries) - set(args.skip)):
if args.file:
file_id = 'file'
elif args.id:
file_id = 'id'
else:
file_id = 'both'
editEntry(filename, file_id)
sys.exit()
elif args.func == 'list':
listPapers = backend.getEntries(full=True)
if not listPapers:
sys.exit()
listPapers = [v["file"] for k, v in listPapers.items()]
listPapers.sort()
for paper in listPapers:
print(paper)
sys.exit()
elif args.func == 'search':
raise Exception('TODO')
elif args.func == 'open':
for filename in args.ids:
if not openFile(filename):
sys.exit("Unable to open file associated " +
"to ident "+filename)
sys.exit()
elif args.func == 'export':
bibtex = ''
for id in args.ids:
bibtex += tools.parsed2Bibtex(backend.getBibtex(id,
clean=True))
print(bibtex.strip())
sys.exit
elif args.func == 'resync':
confirm = tools.rawInput("Resync files and bibtex index? [y/N] ")
if confirm.lower() == 'y':
resync()
sys.exit()
elif args.func == 'update':
if args.entries is None:
entries = backend.getEntries()
else:
entries = args.entries
for entry in entries:
update(entry)
sys.exit()
except KeyboardInterrupt:
sys.exit()