Fix some imports, clean the cozyweboob Python module

Also add a distinction between documents and bills in CapDocument
capability.
This commit is contained in:
Lucas Verney 2016-10-07 11:27:32 -04:00
parent 2263d8c724
commit e35a2fe083
5 changed files with 150 additions and 116 deletions

View File

@ -19,7 +19,7 @@ First, you need to have Weboob installed on your system.
Typical command-line usage for this script is: Typical command-line usage for this script is:
```bash ```bash
cat konnectors.json | ./cozyweboob.py cat konnectors.json | python -m cozyweboob.__main__
``` ```
where `konnectors.json` is a valid JSON file defining konnectors to be used. where `konnectors.json` is a valid JSON file defining konnectors to be used.

118
cozyweboob/WeboobProxy.py Executable file
View File

@ -0,0 +1,118 @@
#!/usr/bin/env python2
"""
Wrapper script around Weboob to be able to use it in combination with Cozy +
Konnectors easily.
Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/)
written by bnjbvr and released under MIT.
"""
from __future__ import print_function
import logging
from weboob.core import Weboob
from weboob.exceptions import ModuleInstallError
from tools.progress import DummyProgress
import tools.weboob_tools as weboob_tools
# Module specific logger
logger = logging.getLogger(__name__)
class WeboobProxy(object):
"""
Connector is a tool that connects to common websites like bank website,
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get Weboob version.
Returns:
the version of installed Weboob.
"""
return Weboob.VERSION
def __init__(self):
"""
Create a Weboob handle.
"""
# Get a weboob instance
self.weboob = Weboob()
self.backend = None
def install_modules(self, capability=None, name=None):
"""
Ensure latest version of modules is installed.
Args:
capability: Restrict the modules to install to a given capability.
name: Only install the specified module.
Returns: A map between name and infos for all installed modules.
"""
repositories = self.weboob.repositories
# Update modules list
repositories.update_repositories(DummyProgress())
# Get module infos
if name:
modules = {name: repositories.get_module_info(name)}
else:
modules = repositories.get_all_modules_info(capability)
# Install modules if required
for infos in modules.values():
if infos is not None and (
not infos.is_installed() or
not infos.is_local()
):
try:
repositories.install(infos, progress=DummyProgress())
except ModuleInstallError as e:
logger.info(str(e))
return {
module_name: dict(infos.dump())
for module_name, infos in modules.items()
if infos.is_installed()
}
def list_modules(self, capability=None, name=None):
"""
Ensure latest version of modules is installed.
Args:
capability: Restrict the modules to install to a given capability.
name: Only install the specified module.
Returns: The list of installed module infos.
"""
# Update modules and get the latest up to date list
installed_modules = self.install_modules(
capability=capability,
name=name
)
# For each module, get back its config options and website base url
for module_name in installed_modules:
module = self.weboob.modules_loader.get_or_load_module(module_name)
installed_modules[module_name]["config"] = (
weboob_tools.dictify_config_desc(module.config)
)
installed_modules[module_name]["website"] = module.website
return installed_modules
def init_backend(self, modulename, parameters):
"""
Backend initialization.
Returns:
the built backend.
"""
# Ensure module is installed
self.install_modules(name=modulename)
# Build backend
self.backend = self.weboob.build_backend(modulename, parameters)
return self.backend

View File

@ -1,2 +1,4 @@
from .cozyweboob import WeboobProxy, main_fetch, main from WeboobProxy import WeboobProxy
from __main__ import main_fetch, main
__all__ = ["WeboobProxy", "main_fetch", "main"] __all__ = ["WeboobProxy", "main_fetch", "main"]

114
cozyweboob/cozyweboob.py → cozyweboob/__main__.py Executable file → Normal file
View File

@ -1,10 +1,4 @@
#!/usr/bin/env python2
""" """
Wrapper script around Weboob to be able to use it in combination with Cozy +
Konnectors easily.
Part of this code comes from [Kresus](https://github.com/bnjbvr/kresus/)
written by bnjbvr and released under MIT.
""" """
from __future__ import print_function from __future__ import print_function
@ -17,117 +11,19 @@ import sys
from getpass import getpass from getpass import getpass
from requests.utils import dict_from_cookiejar from requests.utils import dict_from_cookiejar
from weboob.core import Weboob
from weboob.exceptions import ModuleInstallError
from WeboobProxy import WeboobProxy
from tools.env import is_in_debug_mode from tools.env import is_in_debug_mode
from tools.jsonwriter import pretty_json from tools.jsonwriter import pretty_json
from tools.progress import DummyProgress
import tools.weboob_tools as weboob_tools
# Dynamically load capabilities conversion modules
# Dynamic loading is required to be able to call them programatically.
CAPABILITIES_CONVERSION_MODULES = importlib.import_module("cozyweboob.capabilities")
# Module specific logger # Module specific logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Dynamically load capabilities conversion modules
class WeboobProxy(object): # Dynamic loading is required to be able to call them programatically.
""" CAPABILITIES_CONVERSION_MODULES = importlib.import_module(".capabilities",
Connector is a tool that connects to common websites like bank website, package="cozyweboob")
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get Weboob version.
Returns:
the version of installed Weboob.
"""
return Weboob.VERSION
def __init__(self):
"""
Create a Weboob handle.
"""
# Get a weboob instance
self.weboob = Weboob()
self.backend = None
def install_modules(self, capability=None, name=None):
"""
Ensure latest version of modules is installed.
Args:
capability: Restrict the modules to install to a given capability.
name: Only install the specified module.
Returns: A map between name and infos for all installed modules.
"""
repositories = self.weboob.repositories
# Update modules list
repositories.update_repositories(DummyProgress())
# Get module infos
if name:
modules = {name: repositories.get_module_info(name)}
else:
modules = repositories.get_all_modules_info(capability)
# Install modules if required
for infos in modules.values():
if infos is not None and (
not infos.is_installed() or
not infos.is_local()
):
try:
repositories.install(infos, progress=DummyProgress())
except ModuleInstallError as e:
logger.info(str(e))
return {
module_name: dict(infos.dump())
for module_name, infos in modules.items()
if infos.is_installed()
}
def list_modules(self, capability=None, name=None):
"""
Ensure latest version of modules is installed.
Args:
capability: Restrict the modules to install to a given capability.
name: Only install the specified module.
Returns: The list of installed module infos.
"""
# Update modules and get the latest up to date list
installed_modules = self.install_modules(
capability=capability,
name=name
)
# For each module, get back its config options and website base url
for module_name in installed_modules:
module = self.weboob.modules_loader.get_or_load_module(module_name)
installed_modules[module_name]["config"] = (
weboob_tools.dictify_config_desc(module.config)
)
installed_modules[module_name]["website"] = module.website
return installed_modules
def init_backend(self, modulename, parameters):
"""
Backend initialization.
Returns:
the built backend.
"""
# Ensure module is installed
self.install_modules(name=modulename)
# Build backend
self.backend = self.weboob.build_backend(modulename, parameters)
return self.backend
def main_fetch(used_modules): def main_fetch(used_modules):

View File

@ -3,6 +3,7 @@ This module contains all the conversion functions associated to the Document
capability. capability.
""" """
from base import clean_object from base import clean_object
from weboob.capabilities.bill import Bill
def to_cozy(document): def to_cozy(document):
@ -27,14 +28,30 @@ def to_cozy(document):
# monthly bills for a phone service provider) # monthly bills for a phone service provider)
try: try:
assert subscriptions assert subscriptions
bills = { raw_documents = {
subscription.id: [ subscription.id: list(document.iter_documents(subscription))
clean_object(bill, base_url=base_url)
for bill in document.iter_documents(subscription)
]
for subscription in subscriptions for subscription in subscriptions
} }
raw_bills = {
subscription: [bill for bill in documents if isinstance(bill, Bill)]
for subscription, documents in raw_documents.items()
}
bills = {
subscription: [
clean_object(bill, base_url=base_url)
for bill in bills_list
]
for subscription, bills_list in raw_bills.items()
}
documents = {
subscription: [
clean_object(bill, base_url=base_url)
for bill in documents_list if bill not in raw_bills[subscription]
]
for subscription, documents_list in raw_documents.items()
}
except (NotImplementedError, AssertionError): except (NotImplementedError, AssertionError):
documents = None
bills = None bills = None
# Fetch and clean the list of details of the subscription (detailed # Fetch and clean the list of details of the subscription (detailed
@ -77,5 +94,6 @@ def to_cozy(document):
], ],
"bills": bills, "bills": bills,
"detailed_bills": detailed_bills, "detailed_bills": detailed_bills,
"documents": documents,
"history_bills": history_bills "history_bills": history_bills
} }