Server script and better debugging tools
This commit is contained in:
parent
3e04f3c824
commit
238cf61acc
45
README.md
45
README.md
|
|
@ -15,10 +15,44 @@ communicate with JSON pipes.
|
|||
|
||||
First, you need to have Weboob installed on your system.
|
||||
|
||||
Then, typical command-line usage is:
|
||||
## Cozyweboob script
|
||||
|
||||
Typical command-line usage for this script is:
|
||||
```bash
|
||||
cat konnectors.json | ./cozyweboob.py
|
||||
```
|
||||
where `konnectors.json` is a valid JSON file defining konnectors to be used.
|
||||
|
||||
|
||||
## Server script
|
||||
|
||||
Typical command-line usage for this script is:
|
||||
```bash
|
||||
./server.py
|
||||
```
|
||||
This script spawns a Bottle webserver, listening on `localhost:8080` (by
|
||||
default).
|
||||
|
||||
It has a single route, the index route, which supports `POST` method to send a
|
||||
valid JSON string defining konnectors to be used in a `params` field. Typical
|
||||
example to send it some content is:
|
||||
```bash
|
||||
curl -X POST --data "params=$(cat konnectors.json)" "http://localhost:8080/"
|
||||
```
|
||||
where `konnectors.json` is a valid JSON file defining konnectors to be used.
|
||||
|
||||
Note: You can specify the host and port to listen on using the
|
||||
`COZYWEBOOB_HOST` and `COZYWEBOOB_PORT` environment variables.
|
||||
|
||||
|
||||
## Notes concerning all the available scripts
|
||||
|
||||
Using `COZYWEBOOB_ENV=debug`, you can enable debug features for all of these
|
||||
scripts, which might be useful for development. These features are:
|
||||
* Logging
|
||||
* If you pass a blank field in a JSON konnector description
|
||||
(typically `password: ""`), the script will ask you its value at runtime,
|
||||
using `getpass`.
|
||||
|
||||
|
||||
## Input JSON file
|
||||
|
|
@ -55,6 +89,15 @@ by Weboob. Detailed informations about these other entires can be found in the
|
|||
`doc/capabilities` folder.
|
||||
|
||||
|
||||
## Contributing
|
||||
|
||||
All contributions are welcome. Feel free to make a PR :)
|
||||
|
||||
Python code is currently Python 2, but should be Python 3 compatible as Weboob
|
||||
is moving towards Python 3. All Python code should be PEP8 compliant. I use
|
||||
some extra rules, taken from PyLint.
|
||||
|
||||
|
||||
## License
|
||||
|
||||
The content of this repository is licensed under an MIT license, unless
|
||||
|
|
|
|||
141
cozyweboob.py
141
cozyweboob.py
|
|
@ -18,6 +18,7 @@ import sys
|
|||
from requests.utils import dict_from_cookiejar
|
||||
from weboob.core import Weboob
|
||||
|
||||
from tools.env import is_in_debug_mode
|
||||
from tools.jsonwriter import pretty_json
|
||||
from tools.progress import DummyProgress
|
||||
|
||||
|
|
@ -25,6 +26,9 @@ from tools.progress import DummyProgress
|
|||
# Dynamic loading is required to be able to call them programatically.
|
||||
CAPABILITIES_CONVERSION_MODULES = importlib.import_module("capabilities")
|
||||
|
||||
# Module specific logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WeboobProxy(object):
|
||||
"""
|
||||
|
|
@ -80,88 +84,109 @@ class WeboobProxy(object):
|
|||
return self.backend
|
||||
|
||||
|
||||
def main(used_modules):
|
||||
def mainFetch(used_modules):
|
||||
"""
|
||||
Main code
|
||||
Main fetching code
|
||||
|
||||
Args:
|
||||
used_modules: A list of modules description dicts.
|
||||
Returns: A dict of all the results, ready to be JSON serialized.
|
||||
"""
|
||||
# Update all available modules
|
||||
logging.info("Update all available modules.")
|
||||
logger.info("Update all available modules.")
|
||||
WeboobProxy.update()
|
||||
logging.info("Done updating available modules.")
|
||||
logger.info("Done updating available modules.")
|
||||
|
||||
# Fetch data for the specified modules
|
||||
fetched_data = collections.defaultdict(dict)
|
||||
logging.info("Start fetching from konnectors.")
|
||||
logger.info("Start fetching from konnectors.")
|
||||
for module in used_modules:
|
||||
logging.info("Fetching data from module %s.", module["id"])
|
||||
# Get associated backend for this module
|
||||
backend = WeboobProxy(
|
||||
module["name"],
|
||||
module["parameters"]
|
||||
).get_backend()
|
||||
for capability in backend.iter_caps(): # Supported capabilities
|
||||
# Get capability class name for dynamic import of converter
|
||||
capability = capability.__name__
|
||||
try:
|
||||
fetching_function = (
|
||||
getattr(
|
||||
getattr(
|
||||
CAPABILITIES_CONVERSION_MODULES,
|
||||
capability
|
||||
),
|
||||
"to_cozy"
|
||||
)
|
||||
)
|
||||
logging.info("Fetching capability %s.", capability)
|
||||
# Fetch data and merge them with the ones from other
|
||||
# capabilities
|
||||
fetched_data[module["id"]].update(fetching_function(backend))
|
||||
except AttributeError:
|
||||
# In case the converter does not exist on our side
|
||||
logging.error("%s capability is not implemented.", capability)
|
||||
continue
|
||||
# Store session cookie of this module, to fetch files afterwards
|
||||
try:
|
||||
fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
|
||||
backend.browser.session.cookies
|
||||
)
|
||||
except AttributeError:
|
||||
# Avoid an AttributeError if no session is used for this module
|
||||
fetched_data[module["id"]]["cookies"] = None
|
||||
logging.info("Done fetching from konnectors.")
|
||||
logger.info("Fetching data from module %s.", module["id"])
|
||||
# Get associated backend for this module
|
||||
backend = WeboobProxy(
|
||||
module["name"],
|
||||
module["parameters"]
|
||||
).get_backend()
|
||||
for capability in backend.iter_caps(): # Supported capabilities
|
||||
# Get capability class name for dynamic import of converter
|
||||
capability = capability.__name__
|
||||
try:
|
||||
fetching_function = (
|
||||
getattr(
|
||||
getattr(
|
||||
CAPABILITIES_CONVERSION_MODULES,
|
||||
capability
|
||||
),
|
||||
"to_cozy"
|
||||
)
|
||||
)
|
||||
logger.info("Fetching capability %s.", capability)
|
||||
# Fetch data and merge them with the ones from other
|
||||
# capabilities
|
||||
fetched_data[module["id"]].update(fetching_function(backend))
|
||||
except AttributeError:
|
||||
# In case the converter does not exist on our side
|
||||
logger.error("%s capability is not implemented.", capability)
|
||||
continue
|
||||
# Store session cookie of this module, to fetch files afterwards
|
||||
try:
|
||||
fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
|
||||
backend.browser.session.cookies
|
||||
)
|
||||
except AttributeError:
|
||||
# Avoid an AttributeError if no session is used for this module
|
||||
fetched_data[module["id"]]["cookies"] = None
|
||||
except Exception as e:
|
||||
# Store any error happening in a dedicated field
|
||||
fetched_data[module["id"]]["error"] = e
|
||||
if is_in_debug_mode():
|
||||
# Reraise if in debug
|
||||
raise
|
||||
else:
|
||||
# Skip any errored module when not in debug
|
||||
continue
|
||||
logger.info("Done fetching from konnectors.")
|
||||
return fetched_data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main(json_params):
|
||||
"""
|
||||
Main code
|
||||
|
||||
Args:
|
||||
json_params: A JSON string representing the params to use.
|
||||
Returns: A JSON string of the results.
|
||||
"""
|
||||
try:
|
||||
# Dev: Set logging level and format
|
||||
logging.basicConfig(
|
||||
format='%(levelname)s: %(message)s',
|
||||
level=logging.INFO
|
||||
)
|
||||
try:
|
||||
# Fetch konnectors JSON description from stdin
|
||||
konnectors = json.load(sys.stdin)
|
||||
# Dev: Handle missing passwords using getpass
|
||||
# Fetch konnectors JSON description from stdin
|
||||
konnectors = json.loads(json_params)
|
||||
# Debug only: Handle missing passwords using getpass
|
||||
if is_in_debug_mode():
|
||||
for module in range(len(konnectors)):
|
||||
for param in konnectors[module]["parameters"]:
|
||||
if not konnectors[module]["parameters"][param]:
|
||||
konnectors[module]["parameters"][param] = getpass.getpass(
|
||||
"Password for module %s? " % konnectors[module]["id"]
|
||||
)
|
||||
except ValueError:
|
||||
logging.error("Invalid JSON input.")
|
||||
sys.exit(-1)
|
||||
except ValueError:
|
||||
logger.error("Invalid JSON input.")
|
||||
sys.exit(-1)
|
||||
|
||||
# Output the JSON formatted results on stdout
|
||||
print(
|
||||
pretty_json(
|
||||
main(konnectors)
|
||||
# Output the JSON formatted results on stdout
|
||||
return pretty_json(
|
||||
mainFetch(konnectors)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
# Debug only: Set logging level and format
|
||||
if is_in_debug_mode():
|
||||
logging.basicConfig(
|
||||
format='%(levelname)s: %(message)s',
|
||||
level=logging.INFO
|
||||
)
|
||||
)
|
||||
print(main(sys.stdin.read()))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
bottle
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env python2
|
||||
import os
|
||||
|
||||
from bottle import post, request, run
|
||||
|
||||
from cozyweboob import main as cozyweboob
|
||||
|
||||
|
||||
@post("/")
|
||||
def index():
|
||||
params = request.forms.get("params")
|
||||
return cozyweboob(params)
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get host to listen on
|
||||
host = os.environ.get("COZYWEBOOB_HOST", "localhost")
|
||||
port = os.environ.get("COZYWEBOOB_PORT", 8080)
|
||||
run(host=host, port=port)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
"""
|
||||
Helper functions related to environment variables.
|
||||
"""
|
||||
import os
|
||||
|
||||
|
||||
def is_in_debug_mode():
|
||||
"""
|
||||
Check whether cozyweboob is running in debug mode.
|
||||
|
||||
Returns:
|
||||
true / false
|
||||
"""
|
||||
return "COZYWEBOOB_ENV" in os.environ and os.environ["COZYWEBOOB_ENV"] == "debug"
|
||||
|
|
@ -22,6 +22,9 @@ class CustomJSONEncoder(json.JSONEncoder):
|
|||
elif isinstance(o, Decimal):
|
||||
# Serialize Decimal objects to string
|
||||
return str(o)
|
||||
elif isinstance(o, Exception):
|
||||
# Serialize Exception objects to string representation
|
||||
return repr(o)
|
||||
return json.JSONEncoder.default(self, o)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue