2016-09-29 05:09:14 +02:00
|
|
|
"""
|
2016-10-07 18:29:12 +02:00
|
|
|
Main script for this module
|
2016-09-29 05:09:14 +02:00
|
|
|
"""
|
2016-11-01 04:44:32 +01:00
|
|
|
from __future__ import absolute_import
|
2016-09-28 21:38:55 +02:00
|
|
|
from __future__ import print_function
|
|
|
|
|
2016-09-30 05:03:09 +02:00
|
|
|
import collections
|
2016-09-29 05:09:14 +02:00
|
|
|
import importlib
|
2016-09-28 23:28:29 +02:00
|
|
|
import json
|
2016-09-29 05:09:14 +02:00
|
|
|
import logging
|
2016-11-01 04:44:32 +01:00
|
|
|
import os
|
|
|
|
import shutil
|
2016-09-28 23:28:29 +02:00
|
|
|
import sys
|
2016-11-01 04:44:32 +01:00
|
|
|
import tempfile
|
2016-09-28 21:38:55 +02:00
|
|
|
|
2016-10-03 03:24:01 +02:00
|
|
|
from getpass import getpass
|
|
|
|
|
2016-09-30 05:03:09 +02:00
|
|
|
from requests.utils import dict_from_cookiejar
|
2016-09-28 21:38:55 +02:00
|
|
|
|
2016-10-12 20:27:33 +02:00
|
|
|
from cozyweboob.WeboobProxy import WeboobProxy
|
|
|
|
from cozyweboob.tools.env import is_in_debug_mode
|
|
|
|
from cozyweboob.tools.jsonwriter import pretty_json
|
2016-09-28 21:38:55 +02:00
|
|
|
|
2016-09-29 05:09:14 +02:00
|
|
|
|
2016-09-30 21:43:32 +02:00
|
|
|
# Module specific logger
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
2016-10-07 17:27:32 +02:00
|
|
|
# Dynamically load capabilities conversion modules
|
|
|
|
# Dynamic loading is required to be able to call them programatically.
|
|
|
|
CAPABILITIES_CONVERSION_MODULES = importlib.import_module(".capabilities",
|
|
|
|
package="cozyweboob")
|
2016-09-28 23:28:29 +02:00
|
|
|
|
2016-09-28 21:38:55 +02:00
|
|
|
|
2016-11-01 04:44:32 +01:00
|
|
|
def clean():
|
|
|
|
"""
|
|
|
|
Delete all the temporary downloaded files. These are the
|
|
|
|
"cozyweboob-*-tmp" folders in your system tmp dir.
|
|
|
|
"""
|
|
|
|
sys_tmp_dir = tempfile.gettempdir()
|
|
|
|
tmp_dirs = [
|
|
|
|
x
|
|
|
|
for x in os.listdir(sys_tmp_dir)
|
|
|
|
if os.path.isdir(os.path.join(sys_tmp_dir, x))
|
|
|
|
]
|
|
|
|
removed_dirs = []
|
|
|
|
for tmp_dir in tmp_dirs:
|
|
|
|
if tmp_dir.startswith("cozyweboob-") and tmp_dir.endswith("-tmp"):
|
|
|
|
tmp_dir = os.path.join(sys_tmp_dir, tmp_dir)
|
|
|
|
removed_dirs.append(tmp_dir)
|
|
|
|
shutil.rmtree(tmp_dir)
|
|
|
|
return {
|
|
|
|
"removed_dirs": removed_dirs
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-10-03 03:24:01 +02:00
|
|
|
def main_fetch(used_modules):
|
2016-09-28 21:38:55 +02:00
|
|
|
"""
|
2016-09-30 21:43:32 +02:00
|
|
|
Main fetching code
|
2016-09-30 05:03:09 +02:00
|
|
|
|
|
|
|
Args:
|
|
|
|
used_modules: A list of modules description dicts.
|
|
|
|
Returns: A dict of all the results, ready to be JSON serialized.
|
2016-09-28 21:38:55 +02:00
|
|
|
"""
|
2016-09-28 23:28:29 +02:00
|
|
|
# Fetch data for the specified modules
|
2016-09-30 05:03:09 +02:00
|
|
|
fetched_data = collections.defaultdict(dict)
|
2016-09-30 21:43:32 +02:00
|
|
|
logger.info("Start fetching from konnectors.")
|
2016-09-29 05:09:14 +02:00
|
|
|
for module in used_modules:
|
2016-09-30 21:43:32 +02:00
|
|
|
try:
|
2016-10-03 03:24:01 +02:00
|
|
|
weboob_proxy = WeboobProxy()
|
2016-09-30 21:43:32 +02:00
|
|
|
logger.info("Fetching data from module %s.", module["id"])
|
|
|
|
# Get associated backend for this module
|
2016-10-03 03:24:01 +02:00
|
|
|
backend = weboob_proxy.init_backend(
|
2016-09-30 21:43:32 +02:00
|
|
|
module["name"],
|
|
|
|
module["parameters"]
|
2016-10-03 03:24:01 +02:00
|
|
|
)
|
2016-09-30 21:43:32 +02:00
|
|
|
for capability in backend.iter_caps(): # Supported capabilities
|
|
|
|
# Get capability class name for dynamic import of converter
|
|
|
|
capability = capability.__name__
|
|
|
|
try:
|
|
|
|
fetching_function = (
|
2016-09-29 05:09:14 +02:00
|
|
|
getattr(
|
2016-09-30 21:43:32 +02:00
|
|
|
getattr(
|
|
|
|
CAPABILITIES_CONVERSION_MODULES,
|
|
|
|
capability
|
|
|
|
),
|
|
|
|
"to_cozy"
|
|
|
|
)
|
2016-09-29 05:09:14 +02:00
|
|
|
)
|
2016-09-30 21:43:32 +02:00
|
|
|
logger.info("Fetching capability %s.", capability)
|
|
|
|
# Fetch data and merge them with the ones from other
|
|
|
|
# capabilities
|
2016-10-03 03:24:01 +02:00
|
|
|
fetched_data[module["id"]].update(
|
2016-10-07 18:29:12 +02:00
|
|
|
fetching_function(
|
|
|
|
backend,
|
|
|
|
# If no actions specified, fetch but don't download
|
2016-10-12 20:27:33 +02:00
|
|
|
module.get("actions", {
|
|
|
|
"fetch": True,
|
|
|
|
"download": False
|
|
|
|
})
|
2016-10-07 18:29:12 +02:00
|
|
|
)
|
2016-10-03 03:24:01 +02:00
|
|
|
)
|
2016-09-30 21:43:32 +02:00
|
|
|
except AttributeError:
|
|
|
|
# In case the converter does not exist on our side
|
2016-10-03 03:24:01 +02:00
|
|
|
logger.error("%s capability is not implemented.",
|
|
|
|
capability)
|
2016-09-30 21:43:32 +02:00
|
|
|
continue
|
|
|
|
# Store session cookie of this module, to fetch files afterwards
|
|
|
|
try:
|
|
|
|
fetched_data[module["id"]]["cookies"] = dict_from_cookiejar(
|
|
|
|
backend.browser.session.cookies
|
2016-09-29 05:09:14 +02:00
|
|
|
)
|
|
|
|
except AttributeError:
|
2016-09-30 21:43:32 +02:00
|
|
|
# Avoid an AttributeError if no session is used for this module
|
|
|
|
fetched_data[module["id"]]["cookies"] = None
|
2016-10-12 20:27:33 +02:00
|
|
|
except Exception as exception:
|
2016-09-30 21:43:32 +02:00
|
|
|
# Store any error happening in a dedicated field
|
2016-10-12 20:27:33 +02:00
|
|
|
fetched_data[module["id"]]["error"] = exception
|
2016-09-30 21:43:32 +02:00
|
|
|
if is_in_debug_mode():
|
|
|
|
# Reraise if in debug
|
|
|
|
raise
|
|
|
|
else:
|
|
|
|
# Skip any errored module when not in debug
|
2016-09-29 05:09:14 +02:00
|
|
|
continue
|
2016-09-30 21:43:32 +02:00
|
|
|
logger.info("Done fetching from konnectors.")
|
2016-09-28 23:28:29 +02:00
|
|
|
return fetched_data
|
2016-09-28 21:38:55 +02:00
|
|
|
|
|
|
|
|
2016-09-30 21:43:32 +02:00
|
|
|
def main(json_params):
|
|
|
|
"""
|
|
|
|
Main code
|
|
|
|
|
|
|
|
Args:
|
|
|
|
json_params: A JSON string representing the params to use.
|
|
|
|
Returns: A JSON string of the results.
|
|
|
|
"""
|
2016-09-28 23:28:29 +02:00
|
|
|
try:
|
2016-09-30 21:43:32 +02:00
|
|
|
# Fetch konnectors JSON description from stdin
|
|
|
|
konnectors = json.loads(json_params)
|
|
|
|
# Debug only: Handle missing passwords using getpass
|
|
|
|
if is_in_debug_mode():
|
2016-10-03 03:24:01 +02:00
|
|
|
for module in konnectors:
|
|
|
|
for param in module["parameters"]:
|
|
|
|
if not module["parameters"][param]:
|
|
|
|
module["parameters"][param] = getpass(
|
|
|
|
"Password for module %s? " % (
|
|
|
|
module["id"],
|
|
|
|
)
|
2016-09-29 05:09:14 +02:00
|
|
|
)
|
2016-09-30 21:43:32 +02:00
|
|
|
except ValueError:
|
|
|
|
logger.error("Invalid JSON input.")
|
|
|
|
sys.exit(-1)
|
|
|
|
|
|
|
|
# Output the JSON formatted results on stdout
|
|
|
|
return pretty_json(
|
2016-10-03 03:24:01 +02:00
|
|
|
main_fetch(konnectors)
|
2016-09-30 21:43:32 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
try:
|
|
|
|
# Debug only: Set logging level and format
|
|
|
|
if is_in_debug_mode():
|
|
|
|
logging.basicConfig(
|
|
|
|
format='%(levelname)s: %(message)s',
|
|
|
|
level=logging.INFO
|
2016-09-29 05:09:14 +02:00
|
|
|
)
|
2016-09-30 21:43:32 +02:00
|
|
|
print(main(sys.stdin.read()))
|
2016-09-29 05:09:14 +02:00
|
|
|
except KeyboardInterrupt:
|
|
|
|
pass
|