Add a /list endpoint to the server

Add a new `/list` endpoint to the server. This endpoint lists all the
available modules and their configuration options.

README.md

@@ -41,6 +41,12 @@ curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
 ```
 where `konnectors.json` is a valid JSON file defining konnectors to be used.
 
+
+The server also exposes a `/list` endpoint, which will provide you a JSON dump
+of all the available modules, their descriptions and the configuration options
+you should provide them.
+
+
 Note: You can specify the host and port to listen on using the
 `COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.
 

cozyweboob.py

@@ -18,10 +18,12 @@ from getpass import getpass
 
 from requests.utils import dict_from_cookiejar
 from weboob.core import Weboob
+from weboob.exceptions import ModuleInstallError
 
 from tools.env import is_in_debug_mode
 from tools.jsonwriter import pretty_json
 from tools.progress import DummyProgress
+import tools.weboob_tools as weboob_tools
 
 # Dynamically load capabilities conversion modules
 # Dynamic loading is required to be able to call them programatically.
@@ -50,13 +52,6 @@ class WeboobProxy(object):
         """
         return Weboob.VERSION
 
-    @staticmethod
-    def update():
-        """
-        Ensure modules are up to date.
-        """
-        Weboob().update(progress=DummyProgress())
-
     def __init__(self):
         """
         Create a Weboob handle.
@@ -67,22 +62,59 @@ class WeboobProxy(object):
 
     def install_modules(self, capability=None, name=None):
         """
-        List all available modules and their configuration options.
+        Ensure latest version of modules is installed.
+
+        Args:
+            capability: Restrict the modules to install to a given capability.
+            name: Only install the specified module.
+        Returns: A map between name and infos for all installed modules.
+        """
+        repositories = self.weboob.repositories
+        # Update modules list
+        repositories.update_repositories(DummyProgress())
+        # Get module infos
+        if name:
+            modules = {name: repositories.get_module_info(name)}
+        else:
+            modules = repositories.get_all_modules_info(capability)
+        # Install modules if required
+        for infos in modules.values():
+            if infos is not None and (
+                not infos.is_installed() or
+                not infos.is_local()
+            ):
+                try:
+                    repositories.install(infos, progress=DummyProgress())
+                except ModuleInstallError as e:
+                    logger.info(str(e))
+        return {
+            module_name: dict(infos.dump())
+            for module_name, infos in modules.items()
+            if infos.is_installed()
+        }
+
+    def list_modules(self, capability=None, name=None):
+        """
+        Ensure latest version of modules is installed.
 
         Args:
             capability: Restrict the modules to install to a given capability.
             name: Only install the specified module.
         Returns: The list of installed module infos.
         """
-        repositories = self.weboob.repositories
+        # Update modules and get the latest up to date list
-        if name:
+        installed_modules = self.install_modules(
-            modules = [repositories.get_module_info(name)]
+            capability=capability,
-        else:
+            name=name
-            modules = repositories.get_all_modules_info(capability)
+        )
-        for module in modules:
+        # For each module, get back its config options and website base url
-            if module is not None and not module.is_installed():
+        for module_name in installed_modules:
-                repositories.install(module, progress=DummyProgress())
+            module = self.weboob.modules_loader.get_or_load_module(module_name)
-        return modules
+            installed_modules[module_name]["config"] = (
+                weboob_tools.dictify_config_desc(module.config)
+            )
+            installed_modules[module_name]["website"] = module.website
+        return installed_modules
 
     def init_backend(self, modulename, parameters):
         """
@@ -106,11 +138,6 @@ def main_fetch(used_modules):
         used_modules: A list of modules description dicts.
     Returns: A dict of all the results, ready to be JSON serialized.
     """
-    # Update all available modules
-    logger.info("Update all available modules.")
-    WeboobProxy.update()
-    logger.info("Done updating available modules.")
-
     # Fetch data for the specified modules
     fetched_data = collections.defaultdict(dict)
     logger.info("Start fetching from konnectors.")

server.py

@@ -1,18 +1,62 @@
 #!/usr/bin/env python2
+"""
+HTTP server wrapper around weboob
+"""
+import logging
 import os
 
-from bottle import post, request, run
+from bottle import post, request, response, route, run
 
 from cozyweboob import main as cozyweboob
+from cozyweboob import WeboobProxy
+from tools.env import is_in_debug_mode
+from tools.jsonwriter import pretty_json
+
+# Module specific logger
+logger = logging.getLogger(__name__)
 
 
 @post("/")
 def index():
+    """
+    Main view, fetch from weboob modules.
+    """
     params = request.forms.get("params")
-    return cozyweboob(params)
+    response.content_type = "application/json"
+    return pretty_json(cozyweboob(params))
+
+
+@route("/list")
+def list_view():
+    """
+    List all available weboob modules and their configuration options.
+    """
+    proxy = WeboobProxy()
+    response.content_type = "application/json"
+    return pretty_json(proxy.list_modules())
+
+
+def main():
+    """
+    Main function
+    """
+    # Debug only: Set logging level and format
+    if is_in_debug_mode():
+        logging.basicConfig(
+            format='%(levelname)s: %(message)s',
+            level=logging.INFO
+        )
+    # Ensure all modules are installed and up to date before starting the
+    # server
+    logger.info("Ensuring all modules are installed and up to date.")
+    proxy = WeboobProxy()
+    proxy.install_modules()
+    logger.info("Starting server.")
+    # Get host to listen on
+    HOST = os.environ.get("COZYWEBOOB_HOST", "localhost")
+    PORT = os.environ.get("COZYWEBOOB_PORT", 8080)
+    run(host=HOST, port=PORT, debug=is_in_debug_mode())
+
 
 if __name__ == "__main__":
-    # Get host to listen on
+    main()
-    host = os.environ.get("COZYWEBOOB_HOST", "localhost")
-    port = os.environ.get("COZYWEBOOB_PORT", 8080)
-    run(host=host, port=port)

tools/weboob_tools.py

@@ -0,0 +1,40 @@
+"""
+Helper functions related to Weboob-specific code.
+"""
+from weboob.tools.value import (ValueBackendPassword, ValueInt, ValueFloat,
+                                ValueBool)
+
+
+def value_to_string(value):
+    """
+    Convert a Value definition from Weboob to a string describing the field
+    type.
+
+    Args:
+        value: A Weboob value definition.
+    Returns: A string describing the value type.
+    """
+    if isinstance(value, ValueBackendPassword):
+        return "password"
+    elif isinstance(value, ValueInt):
+        return "int"
+    elif isinstance(value, ValueFloat):
+        return "float"
+    elif isinstance(value, ValueBool):
+        return "bool"
+    else:
+        return "text"
+
+
+def dictify_config_desc(config):
+    """
+    Convert a Weboob BackendConfig description to a JSON-serializabe dict.
+
+    Args:
+        config: A Weboob BackendConfig object.
+    Returns: A JSON-serializable dict.
+    """
+    return {
+        name: value_to_string(value)
+        for name, value in config.items()
+    }