From e35a2fe083c8824ed8cf409679236f9db443cf9d Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Fri, 7 Oct 2016 11:27:32 -0400 Subject: [PATCH] Fix some imports, clean the cozyweboob Python module Also add a distinction between documents and bills in CapDocument capability. --- README.md | 2 +- cozyweboob/WeboobProxy.py | 118 ++++++++++++++++++++++ cozyweboob/__init__.py | 4 +- cozyweboob/{cozyweboob.py => __main__.py} | 114 +-------------------- cozyweboob/capabilities/CapDocument.py | 28 ++++- 5 files changed, 150 insertions(+), 116 deletions(-) create mode 100755 cozyweboob/WeboobProxy.py rename cozyweboob/{cozyweboob.py => __main__.py} (54%) mode change 100755 => 100644 diff --git a/README.md b/README.md index dbb94b2..e351648 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ First, you need to have Weboob installed on your system. Typical command-line usage for this script is: ```bash -cat konnectors.json | ./cozyweboob.py +cat konnectors.json | python -m cozyweboob.__main__ ``` where `konnectors.json` is a valid JSON file defining konnectors to be used. diff --git a/cozyweboob/WeboobProxy.py b/cozyweboob/WeboobProxy.py new file mode 100755 index 0000000..fec40f5 --- /dev/null +++ b/cozyweboob/WeboobProxy.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python2 +""" +Wrapper script around Weboob to be able to use it in combination with Cozy + +Konnectors easily. + +Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/) +written by bnjbvr and released under MIT. +""" +from __future__ import print_function + +import logging + +from weboob.core import Weboob +from weboob.exceptions import ModuleInstallError + +from tools.progress import DummyProgress +import tools.weboob_tools as weboob_tools + + +# Module specific logger +logger = logging.getLogger(__name__) + + +class WeboobProxy(object): + """ + Connector is a tool that connects to common websites like bank website, + phone operator website... and that grabs personal data from there. + Credentials are required to make this operation. + + Technically, connectors are weboob backend wrappers. + """ + + @staticmethod + def version(): + """ + Get Weboob version. + + Returns: + the version of installed Weboob. + """ + return Weboob.VERSION + + def __init__(self): + """ + Create a Weboob handle. + """ + # Get a weboob instance + self.weboob = Weboob() + self.backend = None + + def install_modules(self, capability=None, name=None): + """ + Ensure latest version of modules is installed. + + Args: + capability: Restrict the modules to install to a given capability. + name: Only install the specified module. + Returns: A map between name and infos for all installed modules. + """ + repositories = self.weboob.repositories + # Update modules list + repositories.update_repositories(DummyProgress()) + # Get module infos + if name: + modules = {name: repositories.get_module_info(name)} + else: + modules = repositories.get_all_modules_info(capability) + # Install modules if required + for infos in modules.values(): + if infos is not None and ( + not infos.is_installed() or + not infos.is_local() + ): + try: + repositories.install(infos, progress=DummyProgress()) + except ModuleInstallError as e: + logger.info(str(e)) + return { + module_name: dict(infos.dump()) + for module_name, infos in modules.items() + if infos.is_installed() + } + + def list_modules(self, capability=None, name=None): + """ + Ensure latest version of modules is installed. + + Args: + capability: Restrict the modules to install to a given capability. + name: Only install the specified module. + Returns: The list of installed module infos. + """ + # Update modules and get the latest up to date list + installed_modules = self.install_modules( + capability=capability, + name=name + ) + # For each module, get back its config options and website base url + for module_name in installed_modules: + module = self.weboob.modules_loader.get_or_load_module(module_name) + installed_modules[module_name]["config"] = ( + weboob_tools.dictify_config_desc(module.config) + ) + installed_modules[module_name]["website"] = module.website + return installed_modules + + def init_backend(self, modulename, parameters): + """ + Backend initialization. + + Returns: + the built backend. + """ + # Ensure module is installed + self.install_modules(name=modulename) + # Build backend + self.backend = self.weboob.build_backend(modulename, parameters) + return self.backend diff --git a/cozyweboob/__init__.py b/cozyweboob/__init__.py index 680f889..20eaf57 100644 --- a/cozyweboob/__init__.py +++ b/cozyweboob/__init__.py @@ -1,2 +1,4 @@ -from .cozyweboob import WeboobProxy, main_fetch, main +from WeboobProxy import WeboobProxy +from __main__ import main_fetch, main + __all__ = ["WeboobProxy", "main_fetch", "main"] diff --git a/cozyweboob/cozyweboob.py b/cozyweboob/__main__.py old mode 100755 new mode 100644 similarity index 54% rename from cozyweboob/cozyweboob.py rename to cozyweboob/__main__.py index 9fe1a32..df0c5bf --- a/cozyweboob/cozyweboob.py +++ b/cozyweboob/__main__.py @@ -1,10 +1,4 @@ -#!/usr/bin/env python2 """ -Wrapper script around Weboob to be able to use it in combination with Cozy + -Konnectors easily. - -Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/) -written by bnjbvr and released under MIT. """ from __future__ import print_function @@ -17,117 +11,19 @@ import sys from getpass import getpass from requests.utils import dict_from_cookiejar -from weboob.core import Weboob -from weboob.exceptions import ModuleInstallError +from WeboobProxy import WeboobProxy from tools.env import is_in_debug_mode from tools.jsonwriter import pretty_json -from tools.progress import DummyProgress -import tools.weboob_tools as weboob_tools -# Dynamically load capabilities conversion modules -# Dynamic loading is required to be able to call them programatically. -CAPABILITIES_CONVERSION_MODULES = importlib.import_module("cozyweboob.capabilities") # Module specific logger logger = logging.getLogger(__name__) - -class WeboobProxy(object): - """ - Connector is a tool that connects to common websites like bank website, - phone operator website... and that grabs personal data from there. - Credentials are required to make this operation. - - Technically, connectors are weboob backend wrappers. - """ - - @staticmethod - def version(): - """ - Get Weboob version. - - Returns: - the version of installed Weboob. - """ - return Weboob.VERSION - - def __init__(self): - """ - Create a Weboob handle. - """ - # Get a weboob instance - self.weboob = Weboob() - self.backend = None - - def install_modules(self, capability=None, name=None): - """ - Ensure latest version of modules is installed. - - Args: - capability: Restrict the modules to install to a given capability. - name: Only install the specified module. - Returns: A map between name and infos for all installed modules. - """ - repositories = self.weboob.repositories - # Update modules list - repositories.update_repositories(DummyProgress()) - # Get module infos - if name: - modules = {name: repositories.get_module_info(name)} - else: - modules = repositories.get_all_modules_info(capability) - # Install modules if required - for infos in modules.values(): - if infos is not None and ( - not infos.is_installed() or - not infos.is_local() - ): - try: - repositories.install(infos, progress=DummyProgress()) - except ModuleInstallError as e: - logger.info(str(e)) - return { - module_name: dict(infos.dump()) - for module_name, infos in modules.items() - if infos.is_installed() - } - - def list_modules(self, capability=None, name=None): - """ - Ensure latest version of modules is installed. - - Args: - capability: Restrict the modules to install to a given capability. - name: Only install the specified module. - Returns: The list of installed module infos. - """ - # Update modules and get the latest up to date list - installed_modules = self.install_modules( - capability=capability, - name=name - ) - # For each module, get back its config options and website base url - for module_name in installed_modules: - module = self.weboob.modules_loader.get_or_load_module(module_name) - installed_modules[module_name]["config"] = ( - weboob_tools.dictify_config_desc(module.config) - ) - installed_modules[module_name]["website"] = module.website - return installed_modules - - def init_backend(self, modulename, parameters): - """ - Backend initialization. - - Returns: - the built backend. - """ - # Ensure module is installed - self.install_modules(name=modulename) - # Build backend - self.backend = self.weboob.build_backend(modulename, parameters) - return self.backend +# Dynamically load capabilities conversion modules +# Dynamic loading is required to be able to call them programatically. +CAPABILITIES_CONVERSION_MODULES = importlib.import_module(".capabilities", + package="cozyweboob") def main_fetch(used_modules): diff --git a/cozyweboob/capabilities/CapDocument.py b/cozyweboob/capabilities/CapDocument.py index 8f09f57..366e184 100644 --- a/cozyweboob/capabilities/CapDocument.py +++ b/cozyweboob/capabilities/CapDocument.py @@ -3,6 +3,7 @@ This module contains all the conversion functions associated to the Document capability. """ from base import clean_object +from weboob.capabilities.bill import Bill def to_cozy(document): @@ -27,14 +28,30 @@ def to_cozy(document): # monthly bills for a phone service provider) try: assert subscriptions - bills = { - subscription.id: [ - clean_object(bill, base_url=base_url) - for bill in document.iter_documents(subscription) - ] + raw_documents = { + subscription.id: list(document.iter_documents(subscription)) for subscription in subscriptions } + raw_bills = { + subscription: [bill for bill in documents if isinstance(bill, Bill)] + for subscription, documents in raw_documents.items() + } + bills = { + subscription: [ + clean_object(bill, base_url=base_url) + for bill in bills_list + ] + for subscription, bills_list in raw_bills.items() + } + documents = { + subscription: [ + clean_object(bill, base_url=base_url) + for bill in documents_list if bill not in raw_bills[subscription] + ] + for subscription, documents_list in raw_documents.items() + } except (NotImplementedError, AssertionError): + documents = None bills = None # Fetch and clean the list of details of the subscription (detailed @@ -77,5 +94,6 @@ def to_cozy(document): ], "bills": bills, "detailed_bills": detailed_bills, + "documents": documents, "history_bills": history_bills }