Server script and better debugging tools

README.md

@@ -15,10 +15,44 @@ communicate with JSON pipes.
 
 First, you need to have Weboob installed on your system.
 
-Then, typical command-line usage is:
+## Cozyweboob script
+
+Typical command-line usage for this script is:
 ```bash
 cat konnectors.json | ./cozyweboob.py
 ```
+where `konnectors.json` is a valid JSON file defining konnectors to be used.
+
+
+## Server script
+
+Typical command-line usage for this script is:
+```bash
+./server.py
+```
+This script spawns a Bottle webserver, listening on `localhost:8080` (by
+default).
+
+It has a single route, the index route, which supports `POST` method to send a
+valid JSON string defining konnectors to be used in a `params` field. Typical
+example to send it some content is:
+```bash
+curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
+```
+where `konnectors.json` is a valid JSON file defining konnectors to be used.
+
+Note: You can specify the host and port to listen on using the
+`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.
+
+
+## Notes concerning all the available scripts
+
+Using `COZYWEBOOB_ENV=debug`, you can enable debug features for all of these
+scripts, which might be useful for development. These features are:
+* Logging
+* If you pass a blank field in a JSON konnector description
+(typically `password: ""`), the script will ask you its value at runtime,
+using `getpass`.
 
 
 ## Input JSON file
@@ -55,6 +89,15 @@ by Weboob. Detailed informations about these other entires can be found in the
 `doc/capabilities` folder.
 
 
+## Contributing
+
+All contributions are welcome. Feel free to make a PR :)
+
+Python code is currently Python 2, but should be Python 3 compatible as Weboob
+is moving towards Python 3. All Python code should be PEP8 compliant. I use
+some extra rules, taken from PyLint.
+
+
 ## License
 
 The content of this repository is licensed under an MIT license, unless

cozyweboob.py

@@ -18,6 +18,7 @@ import sys
 from requests.utils import dict_from_cookiejar
 from weboob.core import Weboob
 
+from tools.env import is_in_debug_mode
 from tools.jsonwriter import pretty_json
 from tools.progress import DummyProgress
 
@@ -25,6 +26,9 @@ from tools.progress import DummyProgress
 # Dynamic loading is required to be able to call them programatically.
 CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities")
 
+# Module specific logger
+logger = logging.getLogger(__name__)
+
 
 class WeboobProxy(object):
     """
@@ -80,24 +84,25 @@ class WeboobProxy(object):
         return self.backend
 
 
-def main(used_modules):
+def mainFetch(used_modules):
     """
-    Main code
+    Main fetching code
 
     Args:
         used_modules: A list of modules description dicts.
     Returns: A dict of all the results, ready to be JSON serialized.
     """
     # Update all available modules
-    logging.info("Update all available modules.")
+    logger.info("Update all available modules.")
     WeboobProxy.update()
-    logging.info("Done updating available modules.")
+    logger.info("Done updating available modules.")
 
     # Fetch data for the specified modules
     fetched_data = collections.defaultdict(dict)
-    logging.info("Start fetching from konnectors.")
+    logger.info("Start fetching from konnectors.")
     for module in used_modules:
-        logging.info("Fetching data from module %s.", module["id"])
+        try:
+            logger.info("Fetching data from module %s.", module["id"])
             # Get associated backend for this module
             backend = WeboobProxy(
                 module["name"],
@@ -116,13 +121,13 @@ def main(used_modules):
                             "to_cozy"
                         )
                     )
-                logging.info("Fetching capability %s.", capability)
+                    logger.info("Fetching capability %s.", capability)
                     # Fetch data and merge them with the ones from other
                     # capabilities
                     fetched_data[module["id"]].update(fetching_function(backend))
                 except AttributeError:
                     # In case the converter does not exist on our side
-                logging.error("%s capability is not implemented.", capability)
+                    logger.error("%s capability is not implemented.", capability)
                     continue
             # Store session cookie of this module, to fetch files afterwards
             try:
@@ -132,21 +137,32 @@ def main(used_modules):
             except AttributeError:
                 # Avoid an AttributeError if no session is used for this module
                 fetched_data[module["id"]]["cookies"] = None
-    logging.info("Done fetching from konnectors.")
+        except Exception as e:
+            # Store any error happening in a dedicated field
+            fetched_data[module["id"]]["error"] = e
+            if is_in_debug_mode():
+                # Reraise if in debug
+                raise
+            else:
+                # Skip any errored module when not in debug
+                continue
+    logger.info("Done fetching from konnectors.")
     return fetched_data
 
 
-if __name__ == '__main__':
-    try:
-        # Dev: Set logging level and format
-        logging.basicConfig(
-            format='%(levelname)s: %(message)s',
-            level=logging.INFO
-        )
+def main(json_params):
+    """
+    Main code
+
+    Args:
+        json_params: A JSON string representing the params to use.
+    Returns: A JSON string of the results.
+    """
     try:
         # Fetch konnectors JSON description from stdin
-            konnectors = json.load(sys.stdin)
-            # Dev: Handle missing passwords using getpass
+        konnectors = json.loads(json_params)
+        # Debug only: Handle missing passwords using getpass
+        if is_in_debug_mode():
             for module in range(len(konnectors)):
                 for param in konnectors[module]["parameters"]:
                     if not konnectors[module]["parameters"][param]:
@@ -154,14 +170,23 @@ if __name__ == '__main__':
                             "Password for module %s? " % konnectors[module]["id"]
                         )
     except ValueError:
-            logging.error("Invalid JSON input.")
+        logger.error("Invalid JSON input.")
         sys.exit(-1)
 
     # Output the JSON formatted results on stdout
-        print(
-            pretty_json(
-                main(konnectors)
+    return pretty_json(
+        mainFetch(konnectors)
     )
+
+
+if __name__ == '__main__':
+    try:
+        # Debug only: Set logging level and format
+        if is_in_debug_mode():
+            logging.basicConfig(
+                format='%(levelname)s: %(message)s',
+                level=logging.INFO
             )
+        print(main(sys.stdin.read()))
     except KeyboardInterrupt:
         pass

requirements.txt

@@ -0,0 +1 @@
+bottle

server.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python2
+import os
+
+from bottle import post, request, run
+
+from cozyweboob import main as cozyweboob
+
+
+@post("/")
+def index():
+    params = request.forms.get("params")
+    return cozyweboob(params)
+
+if __name__ == "__main__":
+    # Get host to listen on
+    host = os.environ.get("COZYWEBOOB_HOST", "localhost")
+    port = os.environ.get("COZYWEBOOB_PORT", 8080)
+    run(host=host, port=port)

tools/env.py

@@ -0,0 +1,14 @@
+"""
+Helper functions related to environment variables.
+"""
+import os
+
+
+def is_in_debug_mode():
+    """
+    Check whether cozyweboob is running in debug mode.
+
+    Returns:
+        true / false
+    """
+    return "COZYWEBOOB_ENV" in os.environ and os.environ["COZYWEBOOB_ENV"] == "debug"

tools/jsonwriter.py

@@ -22,6 +22,9 @@ class CustomJSONEncoder(json.JSONEncoder):
         elif isinstance(o, Decimal):
             # Serialize Decimal objects to string
             return str(o)
+        elif isinstance(o, Exception):
+            # Serialize Exception objects to string representation
+            return repr(o)
         return json.JSONEncoder.default(self, o)