Fix some imports, clean the cozyweboob Python module

Also add a distinction between documents and bills in CapDocument
capability.

README.md

@@ -19,7 +19,7 @@ First, you need to have Weboob installed on your system.
 
 Typical command-line usage for this script is:
 ```bash
-cat konnectors.json | ./cozyweboob.py
+cat konnectors.json | python -m cozyweboob.__main__
 ```
 where `konnectors.json` is a valid JSON file defining konnectors to be used.
 

cozyweboob/WeboobProxy.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python2
+"""
+Wrapper script around Weboob to be able to use it in combination with Cozy +
+Konnectors easily.
+
+Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/)
+written by bnjbvr and released under MIT.
+"""
+from __future__ import print_function
+
+import logging
+
+from weboob.core import Weboob
+from weboob.exceptions import ModuleInstallError
+
+from tools.progress import DummyProgress
+import tools.weboob_tools as weboob_tools
+
+
+# Module specific logger
+logger = logging.getLogger(__name__)
+
+
+class WeboobProxy(object):
+    """
+    Connector is a tool that connects to common websites like bank website,
+    phone operator website... and that grabs personal data from there.
+    Credentials are required to make this operation.
+
+    Technically, connectors are weboob backend wrappers.
+    """
+
+    @staticmethod
+    def version():
+        """
+        Get Weboob version.
+
+        Returns:
+            the version of installed Weboob.
+        """
+        return Weboob.VERSION
+
+    def __init__(self):
+        """
+        Create a Weboob handle.
+        """
+        # Get a weboob instance
+        self.weboob = Weboob()
+        self.backend = None
+
+    def install_modules(self, capability=None, name=None):
+        """
+        Ensure latest version of modules is installed.
+
+        Args:
+            capability: Restrict the modules to install to a given capability.
+            name: Only install the specified module.
+        Returns: A map between name and infos for all installed modules.
+        """
+        repositories = self.weboob.repositories
+        # Update modules list
+        repositories.update_repositories(DummyProgress())
+        # Get module infos
+        if name:
+            modules = {name: repositories.get_module_info(name)}
+        else:
+            modules = repositories.get_all_modules_info(capability)
+        # Install modules if required
+        for infos in modules.values():
+            if infos is not None and (
+                not infos.is_installed() or
+                not infos.is_local()
+            ):
+                try:
+                    repositories.install(infos, progress=DummyProgress())
+                except ModuleInstallError as e:
+                    logger.info(str(e))
+        return {
+            module_name: dict(infos.dump())
+            for module_name, infos in modules.items()
+            if infos.is_installed()
+        }
+
+    def list_modules(self, capability=None, name=None):
+        """
+        Ensure latest version of modules is installed.
+
+        Args:
+            capability: Restrict the modules to install to a given capability.
+            name: Only install the specified module.
+        Returns: The list of installed module infos.
+        """
+        # Update modules and get the latest up to date list
+        installed_modules = self.install_modules(
+            capability=capability,
+            name=name
+        )
+        # For each module, get back its config options and website base url
+        for module_name in installed_modules:
+            module = self.weboob.modules_loader.get_or_load_module(module_name)
+            installed_modules[module_name]["config"] = (
+                weboob_tools.dictify_config_desc(module.config)
+            )
+            installed_modules[module_name]["website"] = module.website
+        return installed_modules
+
+    def init_backend(self, modulename, parameters):
+        """
+        Backend initialization.
+
+        Returns:
+            the built backend.
+        """
+        # Ensure module is installed
+        self.install_modules(name=modulename)
+        # Build backend
+        self.backend = self.weboob.build_backend(modulename, parameters)
+        return self.backend

cozyweboob/__init__.py

@@ -1,2 +1,4 @@
-from .cozyweboob import WeboobProxy, main_fetch, main
+from WeboobProxy import WeboobProxy
+from __main__ import main_fetch, main
+
 __all__ = ["WeboobProxy", "main_fetch", "main"]

cozyweboob/__main__.py

@@ -1,10 +1,4 @@
-#!/usr/bin/env python2
 """
-Wrapper script around Weboob to be able to use it in combination with Cozy +
-Konnectors easily.
-
-Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/)
-written by bnjbvr and released under MIT.
 """
 from __future__ import print_function
 
@@ -17,117 +11,19 @@ import sys
 from getpass import getpass
 
 from requests.utils import dict_from_cookiejar
-from weboob.core import Weboob
-from weboob.exceptions import ModuleInstallError
 
+from WeboobProxy import WeboobProxy
 from tools.env import is_in_debug_mode
 from tools.jsonwriter import pretty_json
-from tools.progress import DummyProgress
-import tools.weboob_tools as weboob_tools
 
-# Dynamically load capabilities conversion modules
-# Dynamic loading is required to be able to call them programatically.
-CAPABILITIES_CONVERSION_MODULES = importlib.import_module("cozyweboob.capabilities")
 
 # Module specific logger
 logger = logging.getLogger(__name__)
 
-
+# Dynamically load capabilities conversion modules
-class WeboobProxy(object):
+# Dynamic loading is required to be able to call them programatically.
-    """
+CAPABILITIES_CONVERSION_MODULES = importlib.import_module(".capabilities",
-    Connector is a tool that connects to common websites like bank website,
+                                                          package="cozyweboob")
-    phone operator website... and that grabs personal data from there.
-    Credentials are required to make this operation.
-
-    Technically, connectors are weboob backend wrappers.
-    """
-
-    @staticmethod
-    def version():
-        """
-        Get Weboob version.
-
-        Returns:
-            the version of installed Weboob.
-        """
-        return Weboob.VERSION
-
-    def __init__(self):
-        """
-        Create a Weboob handle.
-        """
-        # Get a weboob instance
-        self.weboob = Weboob()
-        self.backend = None
-
-    def install_modules(self, capability=None, name=None):
-        """
-        Ensure latest version of modules is installed.
-
-        Args:
-            capability: Restrict the modules to install to a given capability.
-            name: Only install the specified module.
-        Returns: A map between name and infos for all installed modules.
-        """
-        repositories = self.weboob.repositories
-        # Update modules list
-        repositories.update_repositories(DummyProgress())
-        # Get module infos
-        if name:
-            modules = {name: repositories.get_module_info(name)}
-        else:
-            modules = repositories.get_all_modules_info(capability)
-        # Install modules if required
-        for infos in modules.values():
-            if infos is not None and (
-                not infos.is_installed() or
-                not infos.is_local()
-            ):
-                try:
-                    repositories.install(infos, progress=DummyProgress())
-                except ModuleInstallError as e:
-                    logger.info(str(e))
-        return {
-            module_name: dict(infos.dump())
-            for module_name, infos in modules.items()
-            if infos.is_installed()
-        }
-
-    def list_modules(self, capability=None, name=None):
-        """
-        Ensure latest version of modules is installed.
-
-        Args:
-            capability: Restrict the modules to install to a given capability.
-            name: Only install the specified module.
-        Returns: The list of installed module infos.
-        """
-        # Update modules and get the latest up to date list
-        installed_modules = self.install_modules(
-            capability=capability,
-            name=name
-        )
-        # For each module, get back its config options and website base url
-        for module_name in installed_modules:
-            module = self.weboob.modules_loader.get_or_load_module(module_name)
-            installed_modules[module_name]["config"] = (
-                weboob_tools.dictify_config_desc(module.config)
-            )
-            installed_modules[module_name]["website"] = module.website
-        return installed_modules
-
-    def init_backend(self, modulename, parameters):
-        """
-        Backend initialization.
-
-        Returns:
-            the built backend.
-        """
-        # Ensure module is installed
-        self.install_modules(name=modulename)
-        # Build backend
-        self.backend = self.weboob.build_backend(modulename, parameters)
-        return self.backend
 
 
 def main_fetch(used_modules):

cozyweboob/capabilities/CapDocument.py

@@ -3,6 +3,7 @@ This module contains all the conversion functions associated to the Document
 capability.
 """
 from base import clean_object
+from weboob.capabilities.bill import Bill
 
 
 def to_cozy(document):
@@ -27,14 +28,30 @@ def to_cozy(document):
     # monthly bills for a phone service provider)
     try:
         assert subscriptions
-        bills = {
+        raw_documents = {
-            subscription.id: [
+            subscription.id: list(document.iter_documents(subscription))
-                clean_object(bill, base_url=base_url)
-                for bill in document.iter_documents(subscription)
-            ]
             for subscription in subscriptions
         }
+        raw_bills = {
+            subscription: [bill for bill in documents if isinstance(bill, Bill)]
+            for subscription, documents in raw_documents.items()
+        }
+        bills = {
+            subscription: [
+                clean_object(bill, base_url=base_url)
+                for bill in bills_list
+            ]
+            for subscription, bills_list in raw_bills.items()
+        }
+        documents = {
+            subscription: [
+                clean_object(bill, base_url=base_url)
+                for bill in documents_list if bill not in raw_bills[subscription]
+            ]
+            for subscription, documents_list in raw_documents.items()
+        }
     except (NotImplementedError, AssertionError):
+        documents = None
         bills = None
 
     # Fetch and clean the list of details of the subscription (detailed
@@ -77,5 +94,6 @@ def to_cozy(document):
         ],
         "bills": bills,
         "detailed_bills": detailed_bills,
+        "documents": documents,
         "history_bills": history_bills
     }