Server script and better debugging tools

This commit is contained in:
Lucas Verney 2016-09-30 15:43:32 -04:00
parent 3e04f3c824
commit 238cf61acc
6 changed files with 163 additions and 59 deletions

View File

@ -15,10 +15,44 @@ communicate with JSON pipes.
First, you need to have Weboob installed on your system. First, you need to have Weboob installed on your system.
Then, typical command-line usage is: ## Cozyweboob script
Typical command-line usage for this script is:
```bash ```bash
cat konnectors.json | ./cozyweboob.py cat konnectors.json | ./cozyweboob.py
``` ```
where `konnectors.json` is a valid JSON file defining konnectors to be used.
## Server script
Typical command-line usage for this script is:
```bash
./server.py
```
This script spawns a Bottle webserver, listening on `localhost:8080` (by
default).
It has a single route, the index route, which supports `POST` method to send a
valid JSON string defining konnectors to be used in a `params` field. Typical
example to send it some content is:
```bash
curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
```
where `konnectors.json` is a valid JSON file defining konnectors to be used.
Note: You can specify the host and port to listen on using the
`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.
## Notes concerning all the available scripts
Using `COZYWEBOOB_ENV=debug`, you can enable debug features for all of these
scripts, which might be useful for development. These features are:
* Logging
* If you pass a blank field in a JSON konnector description
(typically `password: ""`), the script will ask you its value at runtime,
using `getpass`.
## Input JSON file ## Input JSON file
@ -55,6 +89,15 @@ by Weboob. Detailed informations about these other entires can be found in the
`doc/capabilities` folder. `doc/capabilities` folder.
## Contributing
All contributions are welcome. Feel free to make a PR :)
Python code is currently Python 2, but should be Python 3 compatible as Weboob
is moving towards Python 3. All Python code should be PEP8 compliant. I use
some extra rules, taken from PyLint.
## License ## License
The content of this repository is licensed under an MIT license, unless The content of this repository is licensed under an MIT license, unless

View File

@ -18,6 +18,7 @@ import sys
from requests.utils import dict_from_cookiejar from requests.utils import dict_from_cookiejar
from weboob.core import Weboob from weboob.core import Weboob
from tools.env import is_in_debug_mode
from tools.jsonwriter import pretty_json from tools.jsonwriter import pretty_json
from tools.progress import DummyProgress from tools.progress import DummyProgress
@ -25,6 +26,9 @@ from tools.progress import DummyProgress
# Dynamic loading is required to be able to call them programatically. # Dynamic loading is required to be able to call them programatically.
CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities") CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities")
# Module specific logger
logger = logging.getLogger(__name__)
class WeboobProxy(object): class WeboobProxy(object):
""" """
@ -80,24 +84,25 @@ class WeboobProxy(object):
return self.backend return self.backend
def main(used_modules): def mainFetch(used_modules):
""" """
Main code Main fetching code
Args: Args:
used_modules: A list of modules description dicts. used_modules: A list of modules description dicts.
Returns: A dict of all the results, ready to be JSON serialized. Returns: A dict of all the results, ready to be JSON serialized.
""" """
# Update all available modules # Update all available modules
logging.info("Update all available modules.") logger.info("Update all available modules.")
WeboobProxy.update() WeboobProxy.update()
logging.info("Done updating available modules.") logger.info("Done updating available modules.")
# Fetch data for the specified modules # Fetch data for the specified modules
fetched_data = collections.defaultdict(dict) fetched_data = collections.defaultdict(dict)
logging.info("Start fetching from konnectors.") logger.info("Start fetching from konnectors.")
for module in used_modules: for module in used_modules:
logging.info("Fetching data from module %s.", module["id"]) try:
logger.info("Fetching data from module %s.", module["id"])
# Get associated backend for this module # Get associated backend for this module
backend = WeboobProxy( backend = WeboobProxy(
module["name"], module["name"],
@ -116,13 +121,13 @@ def main(used_modules):
"to_cozy" "to_cozy"
) )
) )
logging.info("Fetching capability %s.", capability) logger.info("Fetching capability %s.", capability)
# Fetch data and merge them with the ones from other # Fetch data and merge them with the ones from other
# capabilities # capabilities
fetched_data[module["id"]].update(fetching_function(backend)) fetched_data[module["id"]].update(fetching_function(backend))
except AttributeError: except AttributeError:
# In case the converter does not exist on our side # In case the converter does not exist on our side
logging.error("%s capability is not implemented.", capability) logger.error("%s capability is not implemented.", capability)
continue continue
# Store session cookie of this module, to fetch files afterwards # Store session cookie of this module, to fetch files afterwards
try: try:
@ -132,21 +137,32 @@ def main(used_modules):
except AttributeError: except AttributeError:
# Avoid an AttributeError if no session is used for this module # Avoid an AttributeError if no session is used for this module
fetched_data[module["id"]]["cookies"] = None fetched_data[module["id"]]["cookies"] = None
logging.info("Done fetching from konnectors.") except Exception as e:
# Store any error happening in a dedicated field
fetched_data[module["id"]]["error"] = e
if is_in_debug_mode():
# Reraise if in debug
raise
else:
# Skip any errored module when not in debug
continue
logger.info("Done fetching from konnectors.")
return fetched_data return fetched_data
if __name__ == '__main__': def main(json_params):
try: """
# Dev: Set logging level and format Main code
logging.basicConfig(
format='%(levelname)s: %(message)s', Args:
level=logging.INFO json_params: A JSON string representing the params to use.
) Returns: A JSON string of the results.
"""
try: try:
# Fetch konnectors JSON description from stdin # Fetch konnectors JSON description from stdin
konnectors = json.load(sys.stdin) konnectors = json.loads(json_params)
# Dev: Handle missing passwords using getpass # Debug only: Handle missing passwords using getpass
if is_in_debug_mode():
for module in range(len(konnectors)): for module in range(len(konnectors)):
for param in konnectors[module]["parameters"]: for param in konnectors[module]["parameters"]:
if not konnectors[module]["parameters"][param]: if not konnectors[module]["parameters"][param]:
@ -154,14 +170,23 @@ if __name__ == '__main__':
"Password for module %s? " % konnectors[module]["id"] "Password for module %s? " % konnectors[module]["id"]
) )
except ValueError: except ValueError:
logging.error("Invalid JSON input.") logger.error("Invalid JSON input.")
sys.exit(-1) sys.exit(-1)
# Output the JSON formatted results on stdout # Output the JSON formatted results on stdout
print( return pretty_json(
pretty_json( mainFetch(konnectors)
main(konnectors)
) )
if __name__ == '__main__':
try:
# Debug only: Set logging level and format
if is_in_debug_mode():
logging.basicConfig(
format='%(levelname)s: %(message)s',
level=logging.INFO
) )
print(main(sys.stdin.read()))
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass

1
requirements.txt Normal file
View File

@ -0,0 +1 @@
bottle

18
server.py Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python2
import os
from bottle import post, request, run
from cozyweboob import main as cozyweboob
@post("/")
def index():
params = request.forms.get("params")
return cozyweboob(params)
if __name__ == "__main__":
# Get host to listen on
host = os.environ.get("COZYWEBOOB_HOST", "localhost")
port = os.environ.get("COZYWEBOOB_PORT", 8080)
run(host=host, port=port)

14
tools/env.py Normal file
View File

@ -0,0 +1,14 @@
"""
Helper functions related to environment variables.
"""
import os
def is_in_debug_mode():
"""
Check whether cozyweboob is running in debug mode.
Returns:
true / false
"""
return "COZYWEBOOB_ENV" in os.environ and os.environ["COZYWEBOOB_ENV"] == "debug"

View File

@ -22,6 +22,9 @@ class CustomJSONEncoder(json.JSONEncoder):
elif isinstance(o, Decimal): elif isinstance(o, Decimal):
# Serialize Decimal objects to string # Serialize Decimal objects to string
return str(o) return str(o)
elif isinstance(o, Exception):
# Serialize Exception objects to string representation
return repr(o)
return json.JSONEncoder.default(self, o) return json.JSONEncoder.default(self, o)