From 6fb8a24e4838345982fec722c9799611125530b6 Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Fri, 30 Sep 2016 05:03:09 +0200 Subject: [PATCH] Continue weboob wrapper * Export full absolute URLs in resulting JSON. * Export session cookies in resulting JSON, to download required files on Cozy side. * Add comments in the code. --- .gitignore | 1 + TODO | 6 ---- capabilities/CapDocument.py | 52 +++++++++++++++++++++++++------ capabilities/base.py | 27 ++++++++++++---- cozyweboob.py | 61 ++++++++++++++++++++++++------------- tools/jsonwriter.py | 13 ++++++-- tools/progress.py | 14 +++++++++ 7 files changed, 129 insertions(+), 45 deletions(-) delete mode 100644 TODO create mode 100644 tools/progress.py diff --git a/.gitignore b/.gitignore index fc1b69d..a854523 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc konnectors.json* +out.json diff --git a/TODO b/TODO deleted file mode 100644 index 83fee46..0000000 --- a/TODO +++ /dev/null @@ -1,6 +0,0 @@ -* Bills vs Details? -* Update modules? - -* amazon.com is buggy -* LDLC is out of date -* Bouygues is out of date diff --git a/capabilities/CapDocument.py b/capabilities/CapDocument.py index 6893c4f..d7f80aa 100644 --- a/capabilities/CapDocument.py +++ b/capabilities/CapDocument.py @@ -1,44 +1,76 @@ +""" +This module contains all the conversion functions associated to the Document +capability. +""" from base import clean_object def to_cozy(document): """ - Export a CapDocument object to JSON, to pass it to Cozy instance. + Export a CapDocument object to a JSON-serializable dict, to pass it to Cozy + instance. + + Args: + document: The CapDocument object to handle. + Returns: A JSON-serializable dict for the input object. """ + # Get the BASEURL to generate absolute URLs + base_url = document.browser.BASEURL # Fetch the list of subscriptions try: subscriptions = list(document.iter_subscription()) except NotImplementedError: subscriptions = None + # Fetch and clean the list of bills try: - assert(subscriptions) + assert subscriptions bills = { subscription.id: [ - clean_object(bill) for bill in document.iter_documents(subscription) + clean_object(bill, base_url=base_url) + for bill in document.iter_documents(subscription) ] for subscription in subscriptions } except (NotImplementedError, AssertionError): bills = None - # Fetch and clean the list of history bills (detailed consumption) + + # Fetch and clean the list of details of the subscription (detailed + # consumption) + # TODO: What is this? try: - assert(subscriptions) + assert subscriptions detailed_bills = { subscription.id: [ - clean_object(detailed_bill) + clean_object(detailed_bill, base_url=base_url) for detailed_bill in document.get_details(subscription) ] for subscription in subscriptions } except (NotImplementedError, AssertionError): detailed_bills = None + + # Fetch and clean the list of history bills + try: + assert subscriptions + history_bills = { + subscription.id: [ + clean_object(history_bill, base_url=base_url) + for history_bill in + document.iter_documents_history(subscription) + ] + for subscription in subscriptions + } + except (NotImplementedError, AssertionError): + history_bills = None + # Return a formatted dict with all the infos - ret = { + return { "subscriptions": [ # Clean the subscriptions list - clean_object(subscription) for subscription in subscriptions + clean_object(subscription, base_url=base_url) + for subscription in subscriptions ], "bills": bills, - "detailed_bills": detailed_bills + "detailed_bills": detailed_bills, + "history_bills": history_bills } - return ret diff --git a/capabilities/base.py b/capabilities/base.py index 7968505..ea02e43 100644 --- a/capabilities/base.py +++ b/capabilities/base.py @@ -1,14 +1,29 @@ +""" +Common conversion functions for all the available capabilities. +""" from weboob.capabilities.base import empty -def clean_object(o): +def clean_object(obj, base_url=None): """ - Returns a JSON-serializable dict from a Weboob object. + Helper to get nice JSON-serializable objects from the fields of any Weboob + object deriving from BaseObject. + + Args: + obj: The object to handle. + base_url: An optional base url to generate full URLs in output dict. + Returns: + a JSON-serializable dict for the input object. """ - o = o.to_dict() - for k, v in o.items(): + # Convert object to a dict of its fields + obj = obj.to_dict() + # Clean the various fields to be JSON-serializable + for k, v in obj.items(): if empty(v): # Replace empty values by None, avoid "NotLoaded is not # serializable" error - o[k] = None - return o + obj[k] = None + elif k == "url" and base_url: + # Render full absolute URLs + obj[k] = base_url + v + return obj diff --git a/cozyweboob.py b/cozyweboob.py index e978db4..cc4f3ce 100755 --- a/cozyweboob.py +++ b/cozyweboob.py @@ -1,20 +1,25 @@ #!/usr/bin/env python2 """ -TODO +Wrapper script around Weboob to be able to use it in combination with Cozy + +Konnectors easily. """ from __future__ import print_function +import collections import getpass import importlib import json import logging import sys +from requests.utils import dict_from_cookiejar from weboob.core import Weboob from tools.jsonwriter import pretty_json +from tools.progress import DummyProgress # Dynamically load capabilities conversion modules +# Dynamic loading is required to be able to call them programatically. CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities") @@ -30,7 +35,10 @@ class WeboobProxy(object): @staticmethod def version(): """ - Return Weboob version. + Get Weboob version. + + Returns: + the version of installed Weboob. """ return Weboob.VERSION @@ -39,31 +47,32 @@ class WeboobProxy(object): """ Ensure modules are up to date. """ - return Weboob().update() + Weboob().update() def __init__(self, modulename, parameters): """ Create a Weboob handle and try to load the modules. + + Args: + modulename: the name of the weboob module to use. + parameters: A dict of parameters to pass the weboob module. """ + # Get a weboob instance self.weboob = Weboob() - - # Careful: this is extracted from weboob's code. # Install the module if necessary and hide the progress. - class DummyProgress: - def progress(self, a, b): - pass - repositories = self.weboob.repositories minfo = repositories.get_module_info(modulename) if minfo is not None and not minfo.is_installed(): repositories.install(minfo, progress=DummyProgress()) - - # Calls the backend. + # Build a backend for this module self.backend = self.weboob.build_backend(modulename, parameters) def get_backend(self): """ - Get the built backend. + Backend getter. + + Returns: + the built backend. """ return self.backend @@ -71,12 +80,16 @@ class WeboobProxy(object): def main(used_modules): """ Main code + + Args: + used_modules: A list of modules description dicts. + Returns: A dict of all the results, ready to be JSON serialized. """ # Update all available modules # TODO: WeboobProxy.update() # Fetch data for the specified modules - fetched_data = {} + fetched_data = collections.defaultdict(dict) logging.info("Start fetching from konnectors.") for module in used_modules: logging.info("Fetching data from module %s.", module["id"]) @@ -85,12 +98,10 @@ def main(used_modules): module["name"], module["parameters"] ).get_backend() - # List all supported capabilities - for capability in backend.iter_caps(): - # Convert capability class to string name + for capability in backend.iter_caps(): # Supported capabilities + # Get capability class name for dynamic import of converter capability = capability.__name__ try: - # Get conversion function for this capability fetching_function = ( getattr( getattr( @@ -101,25 +112,32 @@ def main(used_modules): ) ) logging.info("Fetching capability %s.", capability) - # Fetch data and store them - # TODO: Ensure there is no overwrite - fetched_data[module["id"]] = fetching_function(backend) + # Fetch data and merge them with the ones from other + # capabilities + fetched_data[module["id"]].update(fetching_function(backend)) except AttributeError: + # In case the converter does not exist on our side logging.error("%s capability is not implemented.", capability) continue + # Store session cookie of this module, to fetch files afterwards + fetched_data[module["id"]]["cookies"] = dict_from_cookiejar( + backend.browser.session.cookies + ) logging.info("Done fetching from konnectors.") return fetched_data if __name__ == '__main__': try: + # Dev: Set logging level and format logging.basicConfig( format='%(levelname)s: %(message)s', level=logging.INFO ) try: + # Fetch konnectors JSON description from stdin konnectors = json.load(sys.stdin) - # Handle missing passwords using getpass + # Dev: Handle missing passwords using getpass for module in range(len(konnectors)): for param in konnectors[module]["parameters"]: if not konnectors[module]["parameters"][param]: @@ -130,6 +148,7 @@ if __name__ == '__main__': logging.error("Invalid JSON input.") sys.exit(-1) + # Output the JSON formatted results on stdout print( pretty_json( main(konnectors) diff --git a/tools/jsonwriter.py b/tools/jsonwriter.py index ed5d06b..ed8b3c6 100644 --- a/tools/jsonwriter.py +++ b/tools/jsonwriter.py @@ -1,3 +1,7 @@ +""" +This module implements a custom JSON writer to be able to serialize data +returned by Weboob and pretty print the output JSON. +""" import json from datetime import date, datetime @@ -18,10 +22,15 @@ class CustomJSONEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, o) -def pretty_json(foo): +def pretty_json(obj): """ Pretty printing of JSON output, using the custom JSONEncoder. + + Args: + obj: the object to JSON serialize. + Returns: + the pretty printed JSON string. """ - return json.dumps(foo, sort_keys=True, + return json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': '), cls=CustomJSONEncoder) diff --git a/tools/progress.py b/tools/progress.py new file mode 100644 index 0000000..97b232e --- /dev/null +++ b/tools/progress.py @@ -0,0 +1,14 @@ +""" +Miscellaneous progress functions. +""" + + +class DummyProgress: + """ + Dummy progress bar, to disable it. + """ + def progress(self, *args): + """ + Progress function. Do nothing. + """ + pass