Add a /list endpoint to the server

Add a new `/list` endpoint to the server. This endpoint lists all the
available modules and their configuration options.
This commit is contained in:
Lucas Verney 2016-10-04 03:32:03 +02:00
parent 96634a322e
commit 93412281d0
4 changed files with 145 additions and 28 deletions

View File

@ -41,6 +41,12 @@ curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
``` ```
where `konnectors.json` is a valid JSON file defining konnectors to be used. where `konnectors.json` is a valid JSON file defining konnectors to be used.
The server also exposes a `/list` endpoint, which will provide you a JSON dump
of all the available modules, their descriptions and the configuration options
you should provide them.
Note: You can specify the host and port to listen on using the Note: You can specify the host and port to listen on using the
`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables. `COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.

View File

@ -18,10 +18,12 @@ from getpass import getpass
from requests.utils import dict_from_cookiejar from requests.utils import dict_from_cookiejar
from weboob.core import Weboob from weboob.core import Weboob
from weboob.exceptions import ModuleInstallError
from tools.env import is_in_debug_mode from tools.env import is_in_debug_mode
from tools.jsonwriter import pretty_json from tools.jsonwriter import pretty_json
from tools.progress import DummyProgress from tools.progress import DummyProgress
import tools.weboob_tools as weboob_tools
# Dynamically load capabilities conversion modules # Dynamically load capabilities conversion modules
# Dynamic loading is required to be able to call them programatically. # Dynamic loading is required to be able to call them programatically.
@ -50,13 +52,6 @@ class WeboobProxy(object):
""" """
return Weboob.VERSION return Weboob.VERSION
@staticmethod
def update():
"""
Ensure modules are up to date.
"""
Weboob().update(progress=DummyProgress())
def __init__(self): def __init__(self):
""" """
Create a Weboob handle. Create a Weboob handle.
@ -67,22 +62,59 @@ class WeboobProxy(object):
def install_modules(self, capability=None, name=None): def install_modules(self, capability=None, name=None):
""" """
List all available modules and their configuration options. Ensure latest version of modules is installed.
Args:
capability: Restrict the modules to install to a given capability.
name: Only install the specified module.
Returns: A map between name and infos for all installed modules.
"""
repositories = self.weboob.repositories
# Update modules list
repositories.update_repositories(DummyProgress())
# Get module infos
if name:
modules = {name: repositories.get_module_info(name)}
else:
modules = repositories.get_all_modules_info(capability)
# Install modules if required
for infos in modules.values():
if infos is not None and (
not infos.is_installed() or
not infos.is_local()
):
try:
repositories.install(infos, progress=DummyProgress())
except ModuleInstallError as e:
logger.info(str(e))
return {
module_name: dict(infos.dump())
for module_name, infos in modules.items()
if infos.is_installed()
}
def list_modules(self, capability=None, name=None):
"""
Ensure latest version of modules is installed.
Args: Args:
capability: Restrict the modules to install to a given capability. capability: Restrict the modules to install to a given capability.
name: Only install the specified module. name: Only install the specified module.
Returns: The list of installed module infos. Returns: The list of installed module infos.
""" """
repositories = self.weboob.repositories # Update modules and get the latest up to date list
if name: installed_modules = self.install_modules(
modules = [repositories.get_module_info(name)] capability=capability,
else: name=name
modules = repositories.get_all_modules_info(capability) )
for module in modules: # For each module, get back its config options and website base url
if module is not None and not module.is_installed(): for module_name in installed_modules:
repositories.install(module, progress=DummyProgress()) module = self.weboob.modules_loader.get_or_load_module(module_name)
return modules installed_modules[module_name]["config"] = (
weboob_tools.dictify_config_desc(module.config)
)
installed_modules[module_name]["website"] = module.website
return installed_modules
def init_backend(self, modulename, parameters): def init_backend(self, modulename, parameters):
""" """
@ -106,11 +138,6 @@ def main_fetch(used_modules):
used_modules: A list of modules description dicts. used_modules: A list of modules description dicts.
Returns: A dict of all the results, ready to be JSON serialized. Returns: A dict of all the results, ready to be JSON serialized.
""" """
# Update all available modules
logger.info("Update all available modules.")
WeboobProxy.update()
logger.info("Done updating available modules.")
# Fetch data for the specified modules # Fetch data for the specified modules
fetched_data = collections.defaultdict(dict) fetched_data = collections.defaultdict(dict)
logger.info("Start fetching from konnectors.") logger.info("Start fetching from konnectors.")

View File

@ -1,18 +1,62 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
"""
HTTP server wrapper around weboob
"""
import logging
import os import os
from bottle import post, request, run from bottle import post, request, response, route, run
from cozyweboob import main as cozyweboob from cozyweboob import main as cozyweboob
from cozyweboob import WeboobProxy
from tools.env import is_in_debug_mode
from tools.jsonwriter import pretty_json
# Module specific logger
logger = logging.getLogger(__name__)
@post("/") @post("/")
def index(): def index():
"""
Main view, fetch from weboob modules.
"""
params = request.forms.get("params") params = request.forms.get("params")
return cozyweboob(params) response.content_type = "application/json"
return pretty_json(cozyweboob(params))
@route("/list")
def list_view():
"""
List all available weboob modules and their configuration options.
"""
proxy = WeboobProxy()
response.content_type = "application/json"
return pretty_json(proxy.list_modules())
def main():
"""
Main function
"""
# Debug only: Set logging level and format
if is_in_debug_mode():
logging.basicConfig(
format='%(levelname)s: %(message)s',
level=logging.INFO
)
# Ensure all modules are installed and up to date before starting the
# server
logger.info("Ensuring all modules are installed and up to date.")
proxy = WeboobProxy()
proxy.install_modules()
logger.info("Starting server.")
# Get host to listen on
HOST = os.environ.get("COZYWEBOOB_HOST", "localhost")
PORT = os.environ.get("COZYWEBOOB_PORT", 8080)
run(host=HOST, port=PORT, debug=is_in_debug_mode())
if __name__ == "__main__": if __name__ == "__main__":
# Get host to listen on main()
host = os.environ.get("COZYWEBOOB_HOST", "localhost")
port = os.environ.get("COZYWEBOOB_PORT", 8080)
run(host=host, port=port)

40
tools/weboob_tools.py Normal file
View File

@ -0,0 +1,40 @@
"""
Helper functions related to Weboob-specific code.
"""
from weboob.tools.value import (ValueBackendPassword, ValueInt, ValueFloat,
ValueBool)
def value_to_string(value):
"""
Convert a Value definition from Weboob to a string describing the field
type.
Args:
value: A Weboob value definition.
Returns: A string describing the value type.
"""
if isinstance(value, ValueBackendPassword):
return "password"
elif isinstance(value, ValueInt):
return "int"
elif isinstance(value, ValueFloat):
return "float"
elif isinstance(value, ValueBool):
return "bool"
else:
return "text"
def dictify_config_desc(config):
"""
Convert a Weboob BackendConfig description to a JSON-serializabe dict.
Args:
config: A Weboob BackendConfig object.
Returns: A JSON-serializable dict.
"""
return {
name: value_to_string(value)
for name, value in config.items()
}