phyks/bmc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated conf

Browse Source
This commit is contained in:
Phyks 2014-06-30 00:19:38 +02:00
parent bc15f86057
commit 6be5cb3f16
8 changed files with 181 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -5,8 +5,6 @@
*~
.*.sw*
params.py
translation-server
*.pdf
*.bib
*.djvu

11
arxiv2bib.py
View File

@ -184,8 +184,7 @@ class Reference(object):
("Month", self.month),
("Note", self.note),
("Url", self.url),
("File", self.id + ".pdf"),
]:
("File", self.id + ".pdf")]:
if len(v):
lines.append("%-13s = {%s}" % (k, v))
@ -203,12 +202,12 @@ class ReferenceErrorInfo(object):
def bibtex(self):
"""BibTeX comment explaining error"""
return "@comment{%(id)s: %(message)s}" % \
{'id': self.id, 'message': self.message}
return ("@comment{%(id)s: %(message)s}" %
{'id': self.id, 'message': self.message})
def __str__(self):
return "Error: %(message)s (%(id)s)" % \
{'id': self.id, 'message': self.message}
return ("Error: %(message)s (%(id)s)" %
{'id': self.id, 'message': self.message})
def arxiv2bib(id_list):

54
backend.py
View File

@ -14,21 +14,21 @@ import os
import re
import tools
import fetcher
import params
from bibtexparser.bparser import BibTexParser
from config import config
from codecs import open
def getNewName(src, bibtex, tag='', override_format=None):
"""
Return the formatted name according to params for the given
Return the formatted name according to config for the given
bibtex entry
"""
authors = re.split(' and ', bibtex['author'])
if bibtex['type'] == 'article':
if override_format is None:
new_name = params.format_articles
new_name = config.get("format_articles")
else:
new_name = override_format
try:
@ -37,7 +37,7 @@ def getNewName(src, bibtex, tag='', override_format=None):
pass
elif bibtex['type'] == 'book':
if override_format is None:
new_name = params.format_books
new_name = config.get("format_books")
else:
new_name = override_format
@ -53,25 +53,27 @@ def getNewName(src, bibtex, tag='', override_format=None):
if('archiveprefix' in bibtex and
'arXiv' in bibtex['archiveprefix']):
new_name = new_name.replace("%v",
'-'+bibtex['eprint'][bibtex['eprint'].rfind('v'):])
'-' +
bibtex['eprint'][bibtex['eprint'].
rfind('v'):])
else:
new_name = new_name.replace("%v", '')
for custom in params.format_custom:
for custom in config.get("format_custom"):
new_name = custom(new_name)
if tag == '':
new_name = (params.folder + tools.slugify(new_name) +
new_name = (config.get("folder") + tools.slugify(new_name) +
tools.getExtension(src))
else:
if not os.path.isdir(params.folder + tag):
if not os.path.isdir(config.get("folder") + tag):
try:
os.mkdir(params.folder + tag)
os.mkdir(config.get("folder") + tag)
except:
tools.warning("Unable to create tag dir " +
params.folder+tag+".")
config.get("folder")+tag+".")
new_name = (params.folder + tools.slugify(tag) + '/' +
new_name = (config.get("folder") + tools.slugify(tag) + '/' +
tools.slugify(new_name) + tools.getExtension(src))
return new_name
@ -83,7 +85,8 @@ def bibtexAppend(data):
data is a dict for one entry in bibtex, as the one from bibtexparser output
"""
try:
with open(params.folder+'index.bib', 'a', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'a', encoding='utf-8') \
as fh:
fh.write(tools.parsed2Bibtex(data)+"\n")
except:
tools.warning("Unable to open index file.")
@ -94,7 +97,8 @@ def bibtexEdit(ident, modifs):
"""Update ident key in bibtex file, modifications are in modifs dict"""
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read())
bibtex = bibtex.get_entry_dict()
except:
@ -115,7 +119,8 @@ def bibtexRewrite(data):
for entry in data.keys():
bibtex += tools.parsed2Bibtex(data[entry])+"\n"
try:
with open(params.folder+'index.bib', 'w', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'w', encoding='utf-8') \
as fh:
fh.write(bibtex)
except:
tools.warning("Unable to open index file.")
@ -125,7 +130,8 @@ def bibtexRewrite(data):
def deleteId(ident):
"""Delete a file based on its id in the bibtex file"""
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read().decode('utf-8'))
bibtex = bibtex.get_entry_dict()
except:
@ -160,7 +166,8 @@ def deleteId(ident):
def deleteFile(filename):
"""Delete a file based on its filename"""
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read().decode('utf-8'))
bibtex = bibtex.get_entry_dict()
except:
@ -206,10 +213,11 @@ def diffFilesIndex():
* full bibtex entry with file='' if file is not found
* only file entry if file with missing bibtex entry
"""
files = tools.listDir(params.folder)
files = tools.listDir(config.get("folder"))
files = [i for i in files if tools.getExtension(i) in ['.pdf', '.djvu']]
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
index = BibTexParser(fh.read())
index_diff = index.get_entry_dict()
except:
@ -233,10 +241,11 @@ def getBibtex(entry, file_id='both', clean=False):
entry is either a filename or a bibtex ident
file_id is file or id or both to search for a file / id / both
clean is to clean the ignored fields specified in params
clean is to clean the ignored fields specified in config
"""
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read())
bibtex = bibtex.get_entry_dict()
except:
@ -256,7 +265,7 @@ def getBibtex(entry, file_id='both', clean=False):
bibtex_entry = bibtex[key]
break
if clean:
for field in params.ignore_fields:
for field in config.get("ignore_fields"):
try:
del(bibtex_entry[field])
except KeyError:
@ -267,7 +276,8 @@ def getBibtex(entry, file_id='both', clean=False):
def getEntries():
"""Returns the list of all entries in the bibtex index"""
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read())
bibtex = bibtex.get_entry_dict()
except:

38
bmc.py
View File

@ -11,9 +11,9 @@ import backend
import fetcher
import tearpages
import tools
import params
from bibtexparser.bparser import BibTexParser
from codecs import open
from config import config
EDITOR = os.environ.get('EDITOR') if os.environ.get('EDITOR') else 'vim'
@ -116,9 +116,11 @@ def addFile(src, filetype, manual, autoconfirm, tag):
tools.warning("Could not determine the DOI nor the arXiv id nor " +
"the ISBN for "+src+". Switching to manual entry.")
doi_arxiv_isbn = ''
while doi_arxiv_isbn not in ['doi', 'arxiv', 'isbn', 'manual', 'skip']:
doi_arxiv_isbn = tools.rawInput("DOI / arXiv " +
"/ ISBN / manual / skip? ").lower()
while(doi_arxiv_isbn not in
['doi', 'arxiv', 'isbn', 'manual', 'skip']):
doi_arxiv_isbn = (tools.rawInput("DOI / arXiv " +
"/ ISBN / manual / skip? ").
lower())
if doi_arxiv_isbn == 'doi':
doi = tools.rawInput('DOI? ')
elif doi_arxiv_isbn == 'arxiv':
@ -132,7 +134,8 @@ def addFile(src, filetype, manual, autoconfirm, tag):
src+", switching to manual entry.")
doi_arxiv = ''
while doi_arxiv not in ['doi', 'arxiv', 'manual', 'skip']:
doi_arxiv = tools.rawInput("DOI / arXiv / manual / skip? ").lower()
doi_arxiv = (tools.rawInput("DOI / arXiv / manual / skip? ").
lower())
if doi_arxiv == 'doi':
doi = tools.rawInput('DOI? ')
elif doi_arxiv == 'arxiv':
@ -144,7 +147,9 @@ def addFile(src, filetype, manual, autoconfirm, tag):
while isbn_manual not in ['isbn', 'manual', 'skip']:
isbn_manual = tools.rawInput("ISBN / manual / skip? ").lower()
if isbn_manual == 'isbn':
isbn = tools.rawInput('ISBN? ').replace(' ', '').replace('-', '')
isbn = (tools.rawInput('ISBN? ').
replace(' ', '').
replace('-', ''))
elif isbn_manual == 'skip':
return False
elif doi is not False:
@ -200,7 +205,8 @@ def addFile(src, filetype, manual, autoconfirm, tag):
shutil.copy2(src, new_name)
except IOError:
new_name = False
sys.exit("Unable to move file to library dir " + params.folder+".")
sys.exit("Unable to move file to library dir " +
config.get("folder")+".")
# Remove first page of IOP papers
try:
@ -258,7 +264,8 @@ def editEntry(entry, file_id='both'):
os.path.dirname(bibtex['file']))
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
index = BibTexParser(fh.read())
index = index.get_entry_dict()
except:
@ -292,7 +299,8 @@ def downloadFile(url, filetype, manual, autoconfirm, tag):
def openFile(ident):
try:
with open(params.folder+'index.bib', 'r', encoding='utf-8') as fh:
with open(config.get("folder")+'index.bib', 'r', encoding='utf-8') \
as fh:
bibtex = BibTexParser(fh.read())
bibtex = bibtex.get_entry_dict()
except:
@ -363,7 +371,7 @@ def resync():
except IOError:
new_name = False
sys.exit("Unable to move file to library dir " +
params.folder+".")
config.get("folder")+".")
backend.bibtexEdit(entry['id'], {'file': filename})
else:
print("Found file without any associated entry in index:")
@ -388,12 +396,12 @@ def resync():
print(entry['file'] + " removed from disk and " +
"index.")
# Check for empty tag dirs
for i in os.listdir(params.folder):
if os.path.isdir(i) and not os.listdir(params.folder + i):
for i in os.listdir(config.get("folder")):
if os.path.isdir(i) and not os.listdir(config.get("folder") + i):
try:
os.rmdir(params.folder + i)
os.rmdir(config.get("folder") + i)
except:
tools.warning("Found empty tag dir "+params.folder + i +
tools.warning("Found empty tag dir "+config.get("folder") + i +
" but could not delete it.")
@ -584,7 +592,7 @@ if __name__ == '__main__':
sys.exit()
elif args.func == 'list':
listPapers = tools.listDir(params.folder)
listPapers = tools.listDir(config.get("folder"))
listPapers.sort()
for paper in listPapers:

81
config.py Normal file
View File

@ -0,0 +1,81 @@
import os
import errno
import json
import sys
import tools
# List of available options :
# * folder : folder in which papers are stored
# * proxies : list of proxies to use, e.g. ['', "socks5://localhost:4711"]
# * format_articles, format_books : masks to rename files
# * format_custom : list of lambda functions to apply to rename files.
# E.g. : format_custom = [lambda x: x.replace('test', 'some_expr')]
# * ignore_fields : list of fields to ignore when returning bibtex
# Available masks to rename files are
# %f = last name of first author
# %l = last name of last author
# %j = name of the journal
# %Y = published year
# %t = title
# %a = authors
# %v = arXiv version (e.g. '-v1') or nothing if not an arXiv paper
def make_sure_path_exists(path):
try:
os.makedirs(path)
return True
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
else:
return False
class Config():
def __init__(self):
self.config_path = os.path.expanduser("~/.config/")
self.config = {}
self.load()
def get(self, param):
return self.config[param]
def set(self, param, value):
self.config[param] = value
def initialize(self):
self.set("folder", os.path.expanduser("~/Papers/"))
self.set("proxies", [''])
self.set("format_articles", "%f_%l-%j-%Y%v")
self.set("format_books", "%a-%t")
self.set("format_custom", [])
self.set("ignore_fields", ["file", "doi", "tag"])
self.save()
def load(self):
try:
initialized = make_sure_path_exists(self.config_path)
except:
tools.warning("Unable to create ~/.config folder.")
sys.exit(1)
if not initialized:
self.initialize()
else:
try:
with open(self.config_path + "bmc.json", 'r') as fh:
self.config = json.load(fh)
except (ValueError, IOError):
tools.warning("Config file could not be read.")
sys.exit(1)
def save(self):
try:
with open(self.config_path + "bmc.json", 'r') as fh:
fh.write(json.dumps(self.config))
except IOError:
tools.warning("Could not write config file.")
sys.exit(1)
config = Config()

9
fetcher.py
View File

@ -17,8 +17,8 @@ import subprocess
import sys
import arxiv2bib as arxiv_metadata
import tools
import params
from bibtexparser.bparser import BibTexParser
from config import config
from isbntools.dev._fmt import fmtbib
@ -29,7 +29,7 @@ def download(url):
available proxies sequentially. Returns the raw content of the file, or
false if it could not be downloaded.
"""
for proxy in params.proxies:
for proxy in config.get("proxies"):
r_proxy = {
"http": proxy,
"https": proxy,
@ -150,8 +150,9 @@ def findDOI(src):
extractDOI = doi_re.search(extractfull.lower().replace('Œ', '-'))
if not extractDOI:
# PNAS fix
extractDOI = doi_pnas_re.search(extractfull.lower().replace('pnas',
'/pnas'))
extractDOI = doi_pnas_re.search(extractfull.
lower().
replace('pnas', '/pnas'))
if not extractDOI:
# JSB fix
extractDOI = doi_jsb_re.search(extractfull.lower())

32
params.py.example
View File

@ -1,32 +0,0 @@
# -*- coding: utf8 -*-
# The folder in which the papers should be stored
folder = "/home/phyks/Papers/"
if folder[-1] != '/':
folder = folder+'/'
# The various proxies available to retrieve the content
proxies = [
'',
'socks5://localhost:4711'
]
# The mask to rename files
# %f = last name of first author
# %l = last name of last author
# %j = name of the journal
# %Y = published year
# %t = title
# %a = authors
# %v = arXiv version (e.g. '-v1') or nothing if not an arXiv paper
format_articles = "%f_%l-%j-%Y%v"
format_books = "%a-%t"
# Other customization
# List of lambda functions to apply
# E.g. : format_custom = [lambda x: x.replace('test', 'some_expr')]
format_custom = []
# Fields to ignore when exporting Bibtex entries
ignore_fields = ['file', 'doi', 'tag']

43
tests/test_backend.py
View File

@ -11,6 +11,7 @@
import unittest
from backend import *
from bibtexparser.bparser import BibTexParser
from config import config
import os
import shutil
import tempfile
@ -18,7 +19,7 @@ import tempfile
class TestFetcher(unittest.TestCase):
def setUp(self):
params.folder = tempfile.mkdtemp()+"/"
config.set("folder", tempfile.mkdtemp()+"/")
self.bibtex_article_string = """
@article{1303.3130v1,
abstract={We study the role of the dipolar interaction, correctly accounting for the
@ -59,57 +60,57 @@ Lattice},
def test_getNewName_article(self):
self.assertEqual(getNewName("test.pdf", self.bibtex_article),
params.folder+"N_Bartolo_A_Recati-j-2013-v1.pdf")
config.get("folder")+"N_Bartolo_A_Recati-j-2013-v1.pdf")
def test_getNewName_article_override(self):
self.assertEqual(getNewName("test.pdf", self.bibtex_article, override_format="%f"),
params.folder+"N_Bartolo.pdf")
config.get("folder")+"N_Bartolo.pdf")
def test_getNewName_book(self):
self.assertEqual(getNewName("test.pdf", self.bibtex_book),
params.folder+"C_J_Pethick_H_Smith-Bose-Einstein_Condensation_In_Dilute_Gases.pdf")
config.get("folder")+"C_J_Pethick_H_Smith-Bose-Einstein_Condensation_In_Dilute_Gases.pdf")
def test_getNewName_book_override(self):
self.assertEqual(getNewName("test.pdf", self.bibtex_book, override_format="%a"),
params.folder+"C_J_Pethick_H_Smith.pdf")
config.get("folder")+"C_J_Pethick_H_Smith.pdf")
def test_bibtexAppend(self):
bibtexAppend(self.bibtex_article)
with open(params.folder+'index.bib', 'r') as fh:
with open(config.get("folder")+'index.bib', 'r') as fh:
self.assertEqual(fh.read(),
'@article{1303.3130v1,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={/home/phyks/Papers/N_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n')
def test_bibtexEdit(self):
bibtexAppend(self.bibtex_article)
bibtexEdit(self.bibtex_article['id'], {'id': 'bidule'})
with open(params.folder+'index.bib', 'r') as fh:
with open(config.get("folder")+'index.bib', 'r') as fh:
self.assertEqual(fh.read(),
'@article{bidule,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={/home/phyks/Papers/N_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n')
def test_bibtexRewrite(self):
bibtexAppend(self.bibtex_book)
bibtexRewrite({0: self.bibtex_article})
with open(params.folder+'index.bib', 'r') as fh:
with open(config.get("folder")+'index.bib', 'r') as fh:
self.assertEqual(fh.read(),
'@article{1303.3130v1,\n\tabstract={We study the role of the dipolar interaction, correctly accounting for the\nDipolar-Induced Resonance (DIR), in a quasi-one-dimensional system of ultracold\nbosons. We first show how the DIR affects the lowest-energy states of two\nparticles in a harmonic trap. Then, we consider a deep optical lattice loaded\nwith ultracold dipolar bosons. We describe this many-body system using an\natom-dimer extended Bose-Hubbard model. We analyze the impact of the DIR on the\nphase diagram at T=0 by exact diagonalization of a small-sized system. In\nparticular, the resonance strongly modifies the range of parameters for which a\nmass density wave should occur.},\n\tarchiveprefix={arXiv},\n\tauthor={N. Bartolo and D. J. Papoular and L. Barbiero and C. Menotti and A. Recati},\n\teprint={1303.3130v1},\n\tfile={/home/phyks/Papers/N_Bartolo_A_Recati-j-2013.pdf},\n\tlink={http://arxiv.org/abs/1303.3130v1},\n\tmonth={Mar},\n\tprimaryclass={cond-mat.quant-gas},\n\ttag={},\n\ttitle={Dipolar-Induced Resonance for Ultracold Bosons in a Quasi-1D Optical\nLattice},\n\tyear={2013},\n}\n\n\n')
def test_deleteId(self):
self.bibtex_article['file'] = params.folder+'test.pdf'
self.bibtex_article['file'] = config.get("folder")+'test.pdf'
bibtexAppend(self.bibtex_article)
open(params.folder+'test.pdf', 'w').close()
open(config.get("folder")+'test.pdf', 'w').close()
deleteId(self.bibtex_article['id'])
with open(params.folder+'index.bib', 'r') as fh:
with open(config.get("folder")+'index.bib', 'r') as fh:
self.assertEquals(fh.read().strip(), "")
self.assertFalse(os.path.isfile(params.folder+'test.pdf'))
self.assertFalse(os.path.isfile(config.get("folder")+'test.pdf'))
def test_deleteFile(self):
self.bibtex_article['file'] = params.folder+'test.pdf'
self.bibtex_article['file'] = config.get("folder")+'test.pdf'
bibtexAppend(self.bibtex_article)
open(params.folder+'test.pdf', 'w').close()
open(config.get("folder")+'test.pdf', 'w').close()
deleteFile(self.bibtex_article['file'])
with open(params.folder+'index.bib', 'r') as fh:
with open(config.get("folder")+'index.bib', 'r') as fh:
self.assertEquals(fh.read().strip(), "")
self.assertFalse(os.path.isfile(params.folder+'test.pdf'))
self.assertFalse(os.path.isfile(config.get("folder")+'test.pdf'))
def test_diffFilesIndex(self):
# TODO
@ -126,17 +127,17 @@ Lattice},
self.assertEqual(got, self.bibtex_article)
def test_getBibtex_file(self):
self.bibtex_article['file'] = params.folder+'test.pdf'
open(params.folder+'test.pdf', 'w').close()
self.bibtex_article['file'] = config.get("folder")+'test.pdf'
open(config.get("folder")+'test.pdf', 'w').close()
bibtexAppend(self.bibtex_article)
got = getBibtex(self.bibtex_article['file'], file_id='file')
self.assertEqual(got, self.bibtex_article)
def test_getBibtex_clean(self):
params.ignore_fields = ['id', 'abstract']
config.set("ignore_fields", ['id', 'abstract'])
bibtexAppend(self.bibtex_article)
got = getBibtex(self.bibtex_article['id'], clean=True)
for i in params.ignore_fields:
for i in config.get("ignore_fields"):
self.assertNotIn(i, got)
def test_getEntries(self):
@ -153,7 +154,7 @@ Lattice},
return
def tearDown(self):
shutil.rmtree(params.folder)
shutil.rmtree(config.get("folder"))
if __name__ == '__main__':
unittest.main()