Server script and better debugging tools

README.md

@@ -15,10 +15,44 @@ communicate with JSON pipes.
 
 First, you need to have Weboob installed on your system.
 
-Then, typical command-line usage is:
+## Cozyweboob script
+
+Typical command-line usage for this script is:
 ```bash
 cat konnectors.json | ./cozyweboob.py
 ```
+where `konnectors.json` is a valid JSON file defining konnectors to be used.
+
+
+## Server script
+
+Typical command-line usage for this script is:
+```bash
+./server.py
+```
+This script spawns a Bottle webserver, listening on `localhost:8080` (by
+default).
+
+It has a single route, the index route, which supports `POST` method to send a
+valid JSON string defining konnectors to be used in a `params` field. Typical
+example to send it some content is:
+```bash
+curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
+```
+where `konnectors.json` is a valid JSON file defining konnectors to be used.
+
+Note: You can specify the host and port to listen on using the
+`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.
+
+
+## Notes concerning all the available scripts
+
+Using `COZYWEBOOB_ENV=debug`, you can enable debug features for all of these
+scripts, which might be useful for development. These features are:
+* Logging
+* If you pass a blank field in a JSON konnector description
+(typically `password: ""`), the script will ask you its value at runtime,
+using `getpass`.
 
 
 ## Input JSON file
@@ -55,6 +89,15 @@ by Weboob. Detailed informations about these other entires can be found in the
 `doc/capabilities` folder.
 
 
+## Contributing
+
+All contributions are welcome. Feel free to make a PR :)
+
+Python code is currently Python 2, but should be Python 3 compatible as Weboob
+is moving towards Python 3. All Python code should be PEP8 compliant. I use
+some extra rules, taken from PyLint.
+
+
 ## License
 
 The content of this repository is licensed under an MIT license, unless

cozyweboob.py

@@ -18,6 +18,7 @@ import sys
 from requests.utils import dict_from_cookiejar
 from weboob.core import Weboob
 
+from tools.env import is_in_debug_mode
 from tools.jsonwriter import pretty_json
 from tools.progress import DummyProgress
 
@@ -25,6 +26,9 @@ from tools.progress import DummyProgress
 # Dynamic loading is required to be able to call them programatically.
 CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities")
 
+# Module specific logger
+logger = logging.getLogger(__name__)
+
 
 class WeboobProxy(object):
     """
@@ -80,88 +84,109 @@ class WeboobProxy(object):
         return self.backend
 
 
-def main(used_modules):
+def mainFetch(used_modules):
     """
-    Main code
+    Main fetching code
 
     Args:
         used_modules: A list of modules description dicts.
     Returns: A dict of all the results, ready to be JSON serialized.
     """
     # Update all available modules
-    logging.info("Update all available modules.")
+    logger.info("Update all available modules.")
     WeboobProxy.update()
-    logging.info("Done updating available modules.")
+    logger.info("Done updating available modules.")
 
     # Fetch data for the specified modules
     fetched_data = collections.defaultdict(dict)
-    logging.info("Start fetching from konnectors.")
+    logger.info("Start fetching from konnectors.")
     for module in used_modules:
-        logging.info("Fetching data from module %s.", module["id"])
-        # Get associated backend for this module
-        backend = WeboobProxy(
-            module["name"],
-            module["parameters"]
-        ).get_backend()
-        for capability in backend.iter_caps():  # Supported capabilities
-            # Get capability class name for dynamic import of converter
-            capability = capability.__name__
-            try:
-                fetching_function = (
-                    getattr(
-                        getattr(
-                            CAPABILITIES_CONVERSION_MODULES,
-                            capability
-                        ),
-                        "to_cozy"
-                    )
-                )
-                logging.info("Fetching capability %s.", capability)
-                # Fetch data and merge them with the ones from other
-                # capabilities
-                fetched_data[module["id"]].update(fetching_function(backend))
-            except AttributeError:
-                # In case the converter does not exist on our side
-                logging.error("%s capability is not implemented.", capability)
-                continue
-        # Store session cookie of this module, to fetch files afterwards
         try:
-            fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
+            logger.info("Fetching data from module %s.", module["id"])
-                backend.browser.session.cookies
+            # Get associated backend for this module
-            )
+            backend = WeboobProxy(
-        except AttributeError:
+                module["name"],
-            # Avoid an AttributeError if no session is used for this module
+                module["parameters"]
-            fetched_data[module["id"]]["cookies"] = None
+            ).get_backend()
-    logging.info("Done fetching from konnectors.")
+            for capability in backend.iter_caps():  # Supported capabilities
+                # Get capability class name for dynamic import of converter
+                capability = capability.__name__
+                try:
+                    fetching_function = (
+                        getattr(
+                            getattr(
+                                CAPABILITIES_CONVERSION_MODULES,
+                                capability
+                            ),
+                            "to_cozy"
+                        )
+                    )
+                    logger.info("Fetching capability %s.", capability)
+                    # Fetch data and merge them with the ones from other
+                    # capabilities
+                    fetched_data[module["id"]].update(fetching_function(backend))
+                except AttributeError:
+                    # In case the converter does not exist on our side
+                    logger.error("%s capability is not implemented.", capability)
+                    continue
+            # Store session cookie of this module, to fetch files afterwards
+            try:
+                fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
+                    backend.browser.session.cookies
+                )
+            except AttributeError:
+                # Avoid an AttributeError if no session is used for this module
+                fetched_data[module["id"]]["cookies"] = None
+        except Exception as e:
+            # Store any error happening in a dedicated field
+            fetched_data[module["id"]]["error"] = e
+            if is_in_debug_mode():
+                # Reraise if in debug
+                raise
+            else:
+                # Skip any errored module when not in debug
+                continue
+    logger.info("Done fetching from konnectors.")
     return fetched_data
 
 
-if __name__ == '__main__':
+def main(json_params):
+    """
+    Main code
+
+    Args:
+        json_params: A JSON string representing the params to use.
+    Returns: A JSON string of the results.
+    """
     try:
-        # Dev: Set logging level and format
+        # Fetch konnectors JSON description from stdin
-        logging.basicConfig(
+        konnectors = json.loads(json_params)
-            format='%(levelname)s: %(message)s',
+        # Debug only: Handle missing passwords using getpass
-            level=logging.INFO
+        if is_in_debug_mode():
-        )
-        try:
-            # Fetch konnectors JSON description from stdin
-            konnectors = json.load(sys.stdin)
-            # Dev: Handle missing passwords using getpass
             for module in range(len(konnectors)):
                 for param in konnectors[module]["parameters"]:
                     if not konnectors[module]["parameters"][param]:
                         konnectors[module]["parameters"][param] = getpass.getpass(
                             "Password for module %s? " % konnectors[module]["id"]
                         )
-        except ValueError:
+    except ValueError:
-            logging.error("Invalid JSON input.")
+        logger.error("Invalid JSON input.")
-            sys.exit(-1)
+        sys.exit(-1)
 
-        # Output the JSON formatted results on stdout
+    # Output the JSON formatted results on stdout
-        print(
+    return pretty_json(
-            pretty_json(
+        mainFetch(konnectors)
-                main(konnectors)
+    )
+
+
+if __name__ == '__main__':
+    try:
+        # Debug only: Set logging level and format
+        if is_in_debug_mode():
+            logging.basicConfig(
+                format='%(levelname)s: %(message)s',
+                level=logging.INFO
             )
-        )
+        print(main(sys.stdin.read()))
     except KeyboardInterrupt:
         pass

requirements.txt

@@ -0,0 +1 @@
+bottle

server.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python2
+import os
+
+from bottle import post, request, run
+
+from cozyweboob import main as cozyweboob
+
+
+@post("/")
+def index():
+    params = request.forms.get("params")
+    return cozyweboob(params)
+
+if __name__ == "__main__":
+    # Get host to listen on
+    host = os.environ.get("COZYWEBOOB_HOST", "localhost")
+    port = os.environ.get("COZYWEBOOB_PORT", 8080)
+    run(host=host, port=port)

tools/env.py

@@ -0,0 +1,14 @@
+"""
+Helper functions related to environment variables.
+"""
+import os
+
+
+def is_in_debug_mode():
+    """
+    Check whether cozyweboob is running in debug mode.
+
+    Returns:
+        true / false
+    """
+    return "COZYWEBOOB_ENV" in os.environ and os.environ["COZYWEBOOB_ENV"] == "debug"

tools/jsonwriter.py

@@ -22,6 +22,9 @@ class CustomJSONEncoder(json.JSONEncoder):
         elif isinstance(o, Decimal):
             # Serialize Decimal objects to string
             return str(o)
+        elif isinstance(o, Exception):
+            # Serialize Exception objects to string representation
+            return repr(o)
         return json.JSONEncoder.default(self, o)