* Skip option in manual fallback

* Homogeneize manual entry for books / articles
* Sum up skipped elements
* Trailing whitespaces in search
This commit is contained in:
Phyks 2014-06-08 20:40:08 +02:00
parent 059b6c26a4
commit 3591f0cb1f
2 changed files with 55 additions and 25 deletions

46
bmc.py
View File

@ -106,29 +106,37 @@ def addFile(src, filetype, manual, autoconfirm, tag):
tools.warning("Could not determine the DOI nor the arXiv id nor " + tools.warning("Could not determine the DOI nor the arXiv id nor " +
"the ISBN for "+src+". Switching to manual entry.") "the ISBN for "+src+". Switching to manual entry.")
doi_arxiv_isbn = '' doi_arxiv_isbn = ''
while doi_arxiv_isbn not in ['doi', 'arxiv', 'isbn', 'manual']: while doi_arxiv_isbn not in ['doi', 'arxiv', 'isbn', 'manual', 'skip']:
doi_arxiv_isbn = tools.rawInput("DOI / arXiv " + doi_arxiv_isbn = tools.rawInput("DOI / arXiv " +
"/ ISBN / manual? ").lower() "/ ISBN / manual / skip? ").lower()
if doi_arxiv_isbn == 'doi': if doi_arxiv_isbn == 'doi':
doi = tools.rawInput('DOI? ') doi = tools.rawInput('DOI? ')
elif doi_arxiv_isbn == 'arxiv': elif doi_arxiv_isbn == 'arxiv':
arxiv = tools.rawInput('arXiv id? ') arxiv = tools.rawInput('arXiv id? ')
elif doi_arxiv_isbn == 'isbn': elif doi_arxiv_isbn == 'isbn':
isbn = tools.rawInput('ISBN? ') isbn = tools.rawInput('ISBN? ')
elif doi_arxiv_isbn == 'skip':
return False
elif filetype == 'article': elif filetype == 'article':
tools.warning("Could not determine the DOI nor the arXiv id for " + tools.warning("Could not determine the DOI nor the arXiv id for " +
src+", switching to manual entry.") src+", switching to manual entry.")
doi_arxiv = '' doi_arxiv = ''
while doi_arxiv not in ['doi', 'arxiv', 'manual']: while doi_arxiv not in ['doi', 'arxiv', 'manual', 'skip']:
doi_arxiv = tools.rawInput("DOI / arXiv / manual? ").lower() doi_arxiv = tools.rawInput("DOI / arXiv / manual / skip? ").lower()
if doi_arxiv == 'doi': if doi_arxiv == 'doi':
doi = tools.rawInput('DOI? ') doi = tools.rawInput('DOI? ')
elif doi_arxiv == 'arxiv': elif doi_arxiv == 'arxiv':
arxiv = tools.rawInput('arXiv id? ') arxiv = tools.rawInput('arXiv id? ')
elif doi_arxiv == 'skip':
return False
elif filetype == 'book': elif filetype == 'book':
tools.warning("Could not determine the ISBN for "+src + isbn_manual = ''
", switching to manual entry.") while isbn_manual not in ['isbn', 'manual', 'skip']:
isbn = tools.rawInput('ISBN? ') isbn_manual = tools.rawInput("ISBN / manual / skip? ").lower()
if isbn_manual == 'isbn':
isbn = tools.rawInput('ISBN? ')
elif isbn_manual == 'skip':
return False
elif doi is not False: elif doi is not False:
print("DOI for "+src+" is "+doi+".") print("DOI for "+src+" is "+doi+".")
elif arxiv is not False: elif arxiv is not False:
@ -263,6 +271,8 @@ def downloadFile(url, filetype, manual, autoconfirm, tag):
with open(tmp.name, 'w+') as fh: with open(tmp.name, 'w+') as fh:
fh.write(dl) fh.write(dl)
new_name = addFile(tmp.name, filetype, manual, autoconfirm, tag) new_name = addFile(tmp.name, filetype, manual, autoconfirm, tag)
if new_name is False:
return False
tmp.close() tmp.close()
return new_name return new_name
else: else:
@ -475,7 +485,7 @@ if __name__ == '__main__':
parser_export = subparsers.add_parser('export', help="export help") parser_export = subparsers.add_parser('export', help="export help")
parser_export.add_argument('ids', metavar='id', nargs='+', parser_export.add_argument('ids', metavar='id', nargs='+',
help="an identifier") help="an identifier")
parser_export.set_defaults(func='export') parser_export.set_defaults(func='export')
parser_resync = subparsers.add_parser('resync', help="resync help") parser_resync = subparsers.add_parser('resync', help="resync help")
@ -494,6 +504,7 @@ if __name__ == '__main__':
args = parser.parse_args() args = parser.parse_args()
try: try:
if args.func == 'download': if args.func == 'download':
skipped = []
for url in args.url: for url in args.url:
new_name = downloadFile(url, args.type, args.manual, args.y, new_name = downloadFile(url, args.type, args.manual, args.y,
args.tag) args.tag)
@ -501,9 +512,15 @@ if __name__ == '__main__':
print(url+" successfully imported as "+new_name) print(url+" successfully imported as "+new_name)
else: else:
tools.warning("An error occurred while downloading "+url) tools.warning("An error occurred while downloading "+url)
skipped.append(url)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit() sys.exit()
if args.func == 'import': if args.func == 'import':
skipped = []
for filename in list(set(args.file) - set(args.skip)): for filename in list(set(args.file) - set(args.skip)):
new_name = addFile(filename, args.type, args.manual, args.y, new_name = addFile(filename, args.type, args.manual, args.y,
args.tag) args.tag)
@ -513,9 +530,15 @@ if __name__ == '__main__':
else: else:
tools.warning("An error occurred while importing " + tools.warning("An error occurred while importing " +
filename) filename)
skipped.append(filename)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit() sys.exit()
elif args.func == 'delete': elif args.func == 'delete':
skipped = []
for filename in list(set(args.entries) - set(args.skip)): for filename in list(set(args.entries) - set(args.skip)):
if not args.force: if not args.force:
confirm = tools.rawInput("Are you sure you want to " + confirm = tools.rawInput("Are you sure you want to " +
@ -530,6 +553,13 @@ if __name__ == '__main__':
sys.exit(1) sys.exit(1)
print(filename+" successfully deleted.") print(filename+" successfully deleted.")
else:
skipped.append(filename)
if len(skipped) > 0:
print("\nSkipped files:")
for i in skipped:
print(i)
sys.exit() sys.exit()
elif args.func == 'edit': elif args.func == 'edit':

View File

@ -6,7 +6,7 @@ http://pyparsing.wikispaces.com/file/view/searchparser.py/30112816/searchparser.
version 2006-03-09 version 2006-03-09
This search query parser uses the excellent Pyparsing module This search query parser uses the excellent Pyparsing module
(http://pyparsing.sourceforge.net/) to parse search queries by users. (http://pyparsing.sourceforge.net/) to parse search queries by users.
It handles: It handles:
@ -34,7 +34,7 @@ are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this * Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer. list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, * Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution. and/or other materials provided with the distribution.
* Neither the name of Estrate nor the names of its contributors may be used * Neither the name of Estrate nor the names of its contributors may be used
to endorse or promote products derived from this software without specific to endorse or promote products derived from this software without specific
@ -45,10 +45,10 @@ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
CONTRIBUTORS: CONTRIBUTORS:
@ -72,12 +72,12 @@ class SearchQueryParser:
'wordwildcard': self.evaluateWordWildcard, 'wordwildcard': self.evaluateWordWildcard,
} }
self._parser = self.parser() self._parser = self.parser()
def parser(self): def parser(self):
""" """
This function returns a parser. This function returns a parser.
The grammar should be like most full text search engines (Google, Tsearch, Lucene). The grammar should be like most full text search engines (Google, Tsearch, Lucene).
Grammar: Grammar:
- a query consists of alphanumeric words, with an optional '*' wildcard - a query consists of alphanumeric words, with an optional '*' wildcard
at the end of a word at the end of a word
@ -89,20 +89,20 @@ class SearchQueryParser:
- if an operator is missing, use an 'and' operator - if an operator is missing, use an 'and' operator
""" """
operatorOr = Forward() operatorOr = Forward()
operatorWord = Group(Combine(Word(alphanums) + Suppress('*'))).setResultsName('wordwildcard') | \ operatorWord = Group(Combine(Word(alphanums) + Suppress('*'))).setResultsName('wordwildcard') | \
Group(Word(alphanums)).setResultsName('word') Group(Word(alphanums)).setResultsName('word')
operatorQuotesContent = Forward() operatorQuotesContent = Forward()
operatorQuotesContent << ( operatorQuotesContent << (
(operatorWord + operatorQuotesContent) | operatorWord (operatorWord + operatorQuotesContent) | operatorWord
) )
operatorQuotes = Group( operatorQuotes = Group(
Or([Suppress('"') + operatorQuotesContent + Suppress('"'), Or([Suppress('"') + operatorQuotesContent + Suppress('"'),
Suppress('\'') + operatorQuotesContent + Suppress('\'')] Suppress('\'') + operatorQuotesContent + Suppress('\'')]
)).setResultsName("quotes") | operatorWord )).setResultsName("quotes") | operatorWord
operatorParenthesis = Group( operatorParenthesis = Group(
(Suppress("(") + operatorOr + Suppress(")")) (Suppress("(") + operatorOr + Suppress(")"))
).setResultsName("parenthesis") | operatorQuotes ).setResultsName("parenthesis") | operatorQuotes
@ -118,7 +118,7 @@ class SearchQueryParser:
).setResultsName("and") | Group( ).setResultsName("and") | Group(
operatorNot + OneOrMore(~oneOf("and or") + operatorAnd) operatorNot + OneOrMore(~oneOf("and or") + operatorAnd)
).setResultsName("and") | operatorNot) ).setResultsName("and") | operatorNot)
operatorOr << (Group( operatorOr << (Group(
operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr
).setResultsName("or") | operatorAnd) ).setResultsName("or") | operatorAnd)
@ -163,7 +163,7 @@ class SearchQueryParser:
def evaluateWordWildcard(self, argument): def evaluateWordWildcard(self, argument):
return self.GetWordWildcard(argument[0]) return self.GetWordWildcard(argument[0])
def evaluate(self, argument): def evaluate(self, argument):
return self._methods[argument.getName()](argument) return self._methods[argument.getName()](argument)
@ -236,7 +236,7 @@ class ParserTest(SearchQueryParser):
7: 'nothing', 7: 'nothing',
8: 'helper', 8: 'helper',
} }
index = { index = {
'help': set((1, 2, 4, 5)), 'help': set((1, 2, 4, 5)),
'me': set((2,)), 'me': set((2,)),
@ -270,7 +270,7 @@ class ParserTest(SearchQueryParser):
if self.docs[item].count(search_string): if self.docs[item].count(search_string):
result.add(item) result.add(item)
return result return result
def GetNot(self, not_set): def GetNot(self, not_set):
all = set(self.docs.keys()) all = set(self.docs.keys())
return all.difference(not_set) return all.difference(not_set)
@ -290,7 +290,7 @@ class ParserTest(SearchQueryParser):
print '>>>>>>>>>>>>>>>>>>>>>>Test ERROR<<<<<<<<<<<<<<<<<<<<<' print '>>>>>>>>>>>>>>>>>>>>>>Test ERROR<<<<<<<<<<<<<<<<<<<<<'
print '' print ''
return all_ok return all_ok
if __name__=='__main__': if __name__=='__main__':
if ParserTest().Test(): if ParserTest().Test():
print 'All tests OK' print 'All tests OK'