phyks
/
cozyweboob
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Continue weboob wrapper 

* Export full absolute URLs in resulting JSON.
* Export session cookies in resulting JSON, to download required files
on Cozy side.
* Add comments in the code.
This commit is contained in:
Lucas Verney 2016-09-30 05:03:09 +02:00
parent 697edaefa3
commit 6fb8a24e48
7 changed files with 129 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
*.pyc
konnectors.json*
out.json

6
TODO
View File

@ -1,6 +0,0 @@
* Bills vs Details?
* Update modules?
* amazon.com is buggy
* LDLC is out of date
* Bouygues is out of date

52
capabilities/CapDocument.py
View File

@ -1,44 +1,76 @@
"""
This module contains all the conversion functions associated to the Document
capability.
"""
from base import clean_object
def to_cozy(document):
"""
Export a CapDocument object to JSON, to pass it to Cozy instance.
Export a CapDocument object to a JSON-serializable dict, to pass it to Cozy
instance.
Args:
document: The CapDocument object to handle.
Returns: A JSON-serializable dict for the input object.
"""
# Get the BASEURL to generate absolute URLs
base_url = document.browser.BASEURL
# Fetch the list of subscriptions
try:
subscriptions = list(document.iter_subscription())
except NotImplementedError:
subscriptions = None
# Fetch and clean the list of bills
try:
assert(subscriptions)
assert subscriptions
bills = {
subscription.id: [
clean_object(bill) for bill in document.iter_documents(subscription)
clean_object(bill, base_url=base_url)
for bill in document.iter_documents(subscription)
]
for subscription in subscriptions
}
except (NotImplementedError, AssertionError):
bills = None
# Fetch and clean the list of history bills (detailed consumption)
# Fetch and clean the list of details of the subscription (detailed
# consumption)
# TODO: What is this?
try:
assert(subscriptions)
assert subscriptions
detailed_bills = {
subscription.id: [
clean_object(detailed_bill)
clean_object(detailed_bill, base_url=base_url)
for detailed_bill in document.get_details(subscription)
]
for subscription in subscriptions
}
except (NotImplementedError, AssertionError):
detailed_bills = None
# Fetch and clean the list of history bills
try:
assert subscriptions
history_bills = {
subscription.id: [
clean_object(history_bill, base_url=base_url)
for history_bill in
document.iter_documents_history(subscription)
]
for subscription in subscriptions
}
except (NotImplementedError, AssertionError):
history_bills = None
# Return a formatted dict with all the infos
ret = {
return {
"subscriptions": [ # Clean the subscriptions list
clean_object(subscription) for subscription in subscriptions
clean_object(subscription, base_url=base_url)
for subscription in subscriptions
],
"bills": bills,
"detailed_bills": detailed_bills
"detailed_bills": detailed_bills,
"history_bills": history_bills
}
return ret

27
capabilities/base.py
View File

@ -1,14 +1,29 @@
"""
Common conversion functions for all the available capabilities.
"""
from weboob.capabilities.base import empty
def clean_object(o):
def clean_object(obj, base_url=None):
"""
Returns a JSON-serializable dict from a Weboob object.
Helper to get nice JSON-serializable objects from the fields of any Weboob
object deriving from BaseObject.
Args:
obj: The object to handle.
base_url: An optional base url to generate full URLs in output dict.
Returns:
a JSON-serializable dict for the input object.
"""
o = o.to_dict()
for k, v in o.items():
# Convert object to a dict of its fields
obj = obj.to_dict()
# Clean the various fields to be JSON-serializable
for k, v in obj.items():
if empty(v):
# Replace empty values by None, avoid "NotLoaded is not
# serializable" error
o[k] = None
return o
obj[k] = None
elif k == "url" and base_url:
# Render full absolute URLs
obj[k] = base_url + v
return obj

61
cozyweboob.py
View File

@ -1,20 +1,25 @@
#!/usr/bin/env python2
"""
TODO
Wrapper script around Weboob to be able to use it in combination with Cozy +
Konnectors easily.
"""
from __future__ import print_function
import collections
import getpass
import importlib
import json
import logging
import sys
from requests.utils import dict_from_cookiejar
from weboob.core import Weboob
from tools.jsonwriter import pretty_json
from tools.progress import DummyProgress
# Dynamically load capabilities conversion modules
# Dynamic loading is required to be able to call them programatically.
CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities")
@ -30,7 +35,10 @@ class WeboobProxy(object):
@staticmethod
def version():
"""
Return Weboob version.
Get Weboob version.
Returns:
the version of installed Weboob.
"""
return Weboob.VERSION
@ -39,31 +47,32 @@ class WeboobProxy(object):
"""
Ensure modules are up to date.
"""
return Weboob().update()
Weboob().update()
def __init__(self, modulename, parameters):
"""
Create a Weboob handle and try to load the modules.
Args:
modulename: the name of the weboob module to use.
parameters: A dict of parameters to pass the weboob module.
"""
# Get a weboob instance
self.weboob = Weboob()
# Careful: this is extracted from weboob's code.
# Install the module if necessary and hide the progress.
class DummyProgress:
def progress(self, a, b):
pass
repositories = self.weboob.repositories
minfo = repositories.get_module_info(modulename)
if minfo is not None and not minfo.is_installed():
repositories.install(minfo, progress=DummyProgress())
# Calls the backend.
# Build a backend for this module
self.backend = self.weboob.build_backend(modulename, parameters)
def get_backend(self):
"""
Get the built backend.
Backend getter.
Returns:
the built backend.
"""
return self.backend
@ -71,12 +80,16 @@ class WeboobProxy(object):
def main(used_modules):
"""
Main code
Args:
used_modules: A list of modules description dicts.
Returns: A dict of all the results, ready to be JSON serialized.
"""
# Update all available modules
# TODO: WeboobProxy.update()
# Fetch data for the specified modules
fetched_data = {}
fetched_data = collections.defaultdict(dict)
logging.info("Start fetching from konnectors.")
for module in used_modules:
logging.info("Fetching data from module %s.", module["id"])
@ -85,12 +98,10 @@ def main(used_modules):
module["name"],
module["parameters"]
).get_backend()
# List all supported capabilities
for capability in backend.iter_caps():
# Convert capability class to string name
for capability in backend.iter_caps(): # Supported capabilities
# Get capability class name for dynamic import of converter
capability = capability.__name__
try:
# Get conversion function for this capability
fetching_function = (
getattr(
getattr(
@ -101,25 +112,32 @@ def main(used_modules):
)
)
logging.info("Fetching capability %s.", capability)
# Fetch data and store them
# TODO: Ensure there is no overwrite
fetched_data[module["id"]] = fetching_function(backend)
# Fetch data and merge them with the ones from other
# capabilities
fetched_data[module["id"]].update(fetching_function(backend))
except AttributeError:
# In case the converter does not exist on our side
logging.error("%s capability is not implemented.", capability)
continue
# Store session cookie of this module, to fetch files afterwards
fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
backend.browser.session.cookies
)
logging.info("Done fetching from konnectors.")
return fetched_data
if __name__ == '__main__':
try:
# Dev: Set logging level and format
logging.basicConfig(
format='%(levelname)s: %(message)s',
level=logging.INFO
)
try:
# Fetch konnectors JSON description from stdin
konnectors = json.load(sys.stdin)
# Handle missing passwords using getpass
# Dev: Handle missing passwords using getpass
for module in range(len(konnectors)):
for param in konnectors[module]["parameters"]:
if not konnectors[module]["parameters"][param]:
@ -130,6 +148,7 @@ if __name__ == '__main__':
logging.error("Invalid JSON input.")
sys.exit(-1)
# Output the JSON formatted results on stdout
print(
pretty_json(
main(konnectors)

13
tools/jsonwriter.py
View File

@ -1,3 +1,7 @@
"""
This module implements a custom JSON writer to be able to serialize data
returned by Weboob and pretty print the output JSON.
"""
import json
from datetime import date, datetime
@ -18,10 +22,15 @@ class CustomJSONEncoder(json.JSONEncoder):
return json.JSONEncoder.default(self, o)
def pretty_json(foo):
def pretty_json(obj):
"""
Pretty printing of JSON output, using the custom JSONEncoder.
Args:
obj: the object to JSON serialize.
Returns:
the pretty printed JSON string.
"""
return json.dumps(foo, sort_keys=True,
return json.dumps(obj, sort_keys=True,
indent=4, separators=(',', ': '),
cls=CustomJSONEncoder)

14
tools/progress.py Normal file
View File

@ -0,0 +1,14 @@
"""
Miscellaneous progress functions.
"""
class DummyProgress:
"""
Dummy progress bar, to disable it.
"""
def progress(self, *args):
"""
Progress function. Do nothing.
"""
pass