diff --git a/README.md b/README.md index defd426..528decd 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,44 @@ communicate with JSON pipes. First, you need to have Weboob installed on your system. -Then, typical command-line usage is: +## Cozyweboob script + +Typical command-line usage for this script is: ```bash cat konnectors.json | ./cozyweboob.py ``` +where `konnectors.json` is a valid JSON file defining konnectors to be used. + + +## Server script + +Typical command-line usage for this script is: +```bash +./server.py +``` +This script spawns a Bottle webserver, listening on `localhost:8080` (by +default). + +It has a single route, the index route, which supports `POST` method to send a +valid JSON string defining konnectors to be used in a `params` field. Typical +example to send it some content is: +```bash +curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/" +``` +where `konnectors.json` is a valid JSON file defining konnectors to be used. + +Note: You can specify the host and port to listen on using the +`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables. + + +## Notes concerning all the available scripts + +Using `COZYWEBOOB_ENV=debug`, you can enable debug features for all of these +scripts, which might be useful for development. These features are: +* Logging +* If you pass a blank field in a JSON konnector description +(typically `password: ""`), the script will ask you its value at runtime, +using `getpass`. ## Input JSON file @@ -55,6 +89,15 @@ by Weboob. Detailed informations about these other entires can be found in the `doc/capabilities` folder. +## Contributing + +All contributions are welcome. Feel free to make a PR :) + +Python code is currently Python 2, but should be Python 3 compatible as Weboob +is moving towards Python 3. All Python code should be PEP8 compliant. I use +some extra rules, taken from PyLint. + + ## License The content of this repository is licensed under an MIT license, unless diff --git a/cozyweboob.py b/cozyweboob.py index e963c20..bbdb98f 100755 --- a/cozyweboob.py +++ b/cozyweboob.py @@ -18,6 +18,7 @@ import sys from requests.utils import dict_from_cookiejar from weboob.core import Weboob +from tools.env import is_in_debug_mode from tools.jsonwriter import pretty_json from tools.progress import DummyProgress @@ -25,6 +26,9 @@ from tools.progress import DummyProgress # Dynamic loading is required to be able to call them programatically. CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities") +# Module specific logger +logger = logging.getLogger(__name__) + class WeboobProxy(object): """ @@ -80,88 +84,109 @@ class WeboobProxy(object): return self.backend -def main(used_modules): +def mainFetch(used_modules): """ - Main code + Main fetching code Args: used_modules: A list of modules description dicts. Returns: A dict of all the results, ready to be JSON serialized. """ # Update all available modules - logging.info("Update all available modules.") + logger.info("Update all available modules.") WeboobProxy.update() - logging.info("Done updating available modules.") + logger.info("Done updating available modules.") # Fetch data for the specified modules fetched_data = collections.defaultdict(dict) - logging.info("Start fetching from konnectors.") + logger.info("Start fetching from konnectors.") for module in used_modules: - logging.info("Fetching data from module %s.", module["id"]) - # Get associated backend for this module - backend = WeboobProxy( - module["name"], - module["parameters"] - ).get_backend() - for capability in backend.iter_caps(): # Supported capabilities - # Get capability class name for dynamic import of converter - capability = capability.__name__ - try: - fetching_function = ( - getattr( - getattr( - CAPABILITIES_CONVERSION_MODULES, - capability - ), - "to_cozy" - ) - ) - logging.info("Fetching capability %s.", capability) - # Fetch data and merge them with the ones from other - # capabilities - fetched_data[module["id"]].update(fetching_function(backend)) - except AttributeError: - # In case the converter does not exist on our side - logging.error("%s capability is not implemented.", capability) - continue - # Store session cookie of this module, to fetch files afterwards try: - fetched_data[module["id"]]["cookies"] = dict_from_cookiejar( - backend.browser.session.cookies - ) - except AttributeError: - # Avoid an AttributeError if no session is used for this module - fetched_data[module["id"]]["cookies"] = None - logging.info("Done fetching from konnectors.") + logger.info("Fetching data from module %s.", module["id"]) + # Get associated backend for this module + backend = WeboobProxy( + module["name"], + module["parameters"] + ).get_backend() + for capability in backend.iter_caps(): # Supported capabilities + # Get capability class name for dynamic import of converter + capability = capability.__name__ + try: + fetching_function = ( + getattr( + getattr( + CAPABILITIES_CONVERSION_MODULES, + capability + ), + "to_cozy" + ) + ) + logger.info("Fetching capability %s.", capability) + # Fetch data and merge them with the ones from other + # capabilities + fetched_data[module["id"]].update(fetching_function(backend)) + except AttributeError: + # In case the converter does not exist on our side + logger.error("%s capability is not implemented.", capability) + continue + # Store session cookie of this module, to fetch files afterwards + try: + fetched_data[module["id"]]["cookies"] = dict_from_cookiejar( + backend.browser.session.cookies + ) + except AttributeError: + # Avoid an AttributeError if no session is used for this module + fetched_data[module["id"]]["cookies"] = None + except Exception as e: + # Store any error happening in a dedicated field + fetched_data[module["id"]]["error"] = e + if is_in_debug_mode(): + # Reraise if in debug + raise + else: + # Skip any errored module when not in debug + continue + logger.info("Done fetching from konnectors.") return fetched_data -if __name__ == '__main__': +def main(json_params): + """ + Main code + + Args: + json_params: A JSON string representing the params to use. + Returns: A JSON string of the results. + """ try: - # Dev: Set logging level and format - logging.basicConfig( - format='%(levelname)s: %(message)s', - level=logging.INFO - ) - try: - # Fetch konnectors JSON description from stdin - konnectors = json.load(sys.stdin) - # Dev: Handle missing passwords using getpass + # Fetch konnectors JSON description from stdin + konnectors = json.loads(json_params) + # Debug only: Handle missing passwords using getpass + if is_in_debug_mode(): for module in range(len(konnectors)): for param in konnectors[module]["parameters"]: if not konnectors[module]["parameters"][param]: konnectors[module]["parameters"][param] = getpass.getpass( "Password for module %s? " % konnectors[module]["id"] ) - except ValueError: - logging.error("Invalid JSON input.") - sys.exit(-1) + except ValueError: + logger.error("Invalid JSON input.") + sys.exit(-1) - # Output the JSON formatted results on stdout - print( - pretty_json( - main(konnectors) + # Output the JSON formatted results on stdout + return pretty_json( + mainFetch(konnectors) + ) + + +if __name__ == '__main__': + try: + # Debug only: Set logging level and format + if is_in_debug_mode(): + logging.basicConfig( + format='%(levelname)s: %(message)s', + level=logging.INFO ) - ) + print(main(sys.stdin.read())) except KeyboardInterrupt: pass diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..310dc0b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +bottle diff --git a/server.py b/server.py new file mode 100755 index 0000000..790dd83 --- /dev/null +++ b/server.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python2 +import os + +from bottle import post, request, run + +from cozyweboob import main as cozyweboob + + +@post("/") +def index(): + params = request.forms.get("params") + return cozyweboob(params) + +if __name__ == "__main__": + # Get host to listen on + host = os.environ.get("COZYWEBOOB_HOST", "localhost") + port = os.environ.get("COZYWEBOOB_PORT", 8080) + run(host=host, port=port) diff --git a/tools/env.py b/tools/env.py new file mode 100644 index 0000000..a2db752 --- /dev/null +++ b/tools/env.py @@ -0,0 +1,14 @@ +""" +Helper functions related to environment variables. +""" +import os + + +def is_in_debug_mode(): + """ + Check whether cozyweboob is running in debug mode. + + Returns: + true / false + """ + return "COZYWEBOOB_ENV" in os.environ and os.environ["COZYWEBOOB_ENV"] == "debug" diff --git a/tools/jsonwriter.py b/tools/jsonwriter.py index a551cfb..ac50ee7 100644 --- a/tools/jsonwriter.py +++ b/tools/jsonwriter.py @@ -22,6 +22,9 @@ class CustomJSONEncoder(json.JSONEncoder): elif isinstance(o, Decimal): # Serialize Decimal objects to string return str(o) + elif isinstance(o, Exception): + # Serialize Exception objects to string representation + return repr(o) return json.JSONEncoder.default(self, o)