Do not store personal data by default, let user overload this behavior

through config.

Fixes #80.
This commit is contained in:
Lucas Verney 2017-11-17 16:14:11 +01:00
parent c936228726
commit 4a65f52475
3 changed files with 25 additions and 5 deletions

View File

@ -113,6 +113,10 @@ List of configuration options:
doc](http://bottlepy.org/docs/dev/deployment.html). doc](http://bottlepy.org/docs/dev/deployment.html).
* `backends` is a list of Weboob backends to enable. It defaults to any * `backends` is a list of Weboob backends to enable. It defaults to any
available and supported Weboob backend. available and supported Weboob backend.
* `store_personal_data` is a boolean indicated whether or not Flatisfy should
fetch personal data from housing posts and store them in database. Such
personal data include contact phone number for instance. By default,
Flatisfy does not store such personal data.
_Note:_ In production, you can either use the `serve` command with a reliable _Note:_ In production, you can either use the `serve` command with a reliable
webserver instead of the default Bottle webserver (specifying a `webserver` webserver instead of the default Bottle webserver (specifying a `webserver`

View File

@ -41,6 +41,9 @@ DEFAULT_CONFIG = {
# Time is in seconds # Time is in seconds
} }
}, },
# Whether or not to store personal data from housing posts (phone number
# etc)
"store_personal_data": False,
# Navitia API key # Navitia API key
"navitia_api_key": None, "navitia_api_key": None,
# Number of filtering passes to run # Number of filtering passes to run
@ -133,6 +136,8 @@ def validate_config(config, check_with_data):
assert config["smtp_port"] is None or isinstance(config["smtp_port"], int) # noqa: E501 assert config["smtp_port"] is None or isinstance(config["smtp_port"], int) # noqa: E501
assert config["smtp_to"] is None or isinstance(config["smtp_to"], list) assert config["smtp_to"] is None or isinstance(config["smtp_to"], list)
assert isinstance(config["store_personal_data"], bool)
# Ensure constraints are ok # Ensure constraints are ok
assert config["constraints"] assert config["constraints"]
for constraint in config["constraints"].values(): for constraint in config["constraints"].values():

View File

@ -173,12 +173,14 @@ class WeboobProxy(object):
return queries return queries
def query(self, query, max_entries=None): def query(self, query, max_entries=None, store_personal_data=False):
""" """
Fetch the housings posts matching a given Weboob query. Fetch the housings posts matching a given Weboob query.
:param query: A Weboob `weboob.capabilities.housing.Query`` object. :param query: A Weboob `weboob.capabilities.housing.Query`` object.
:param max_entries: Maximum number of entries to fetch. :param max_entries: Maximum number of entries to fetch.
:param store_personal_data: Whether personal data should be fetched
from housing posts (phone number etc).
:return: The matching housing posts, dumped as a list of JSON objects. :return: The matching housing posts, dumped as a list of JSON objects.
""" """
housings = [] housings = []
@ -188,6 +190,8 @@ class WeboobProxy(object):
self.webnip.do('search_housings', query), self.webnip.do('search_housings', query),
max_entries max_entries
): ):
if not store_personal_data:
housing.phone = None
housings.append(json.dumps(housing, cls=WeboobEncoder)) housings.append(json.dumps(housing, cls=WeboobEncoder))
except CallErrors as exc: except CallErrors as exc:
# If an error occured, just log it # If an error occured, just log it
@ -197,12 +201,14 @@ class WeboobProxy(object):
) )
return housings return housings
def info(self, full_flat_id): def info(self, full_flat_id, store_personal_data=False):
""" """
Get information (details) about an housing post. Get information (details) about an housing post.
:param full_flat_id: A Weboob housing post id, in complete form :param full_flat_id: A Weboob housing post id, in complete form
(ID@BACKEND) (ID@BACKEND)
:param store_personal_data: Whether personal data should be fetched
from housing posts (phone number etc).
:return: The details in JSON. :return: The details in JSON.
""" """
flat_id, backend_name = full_flat_id.rsplit("@", 1) flat_id, backend_name = full_flat_id.rsplit("@", 1)
@ -220,6 +226,9 @@ class WeboobProxy(object):
housing = backend.get_housing(flat_id) housing = backend.get_housing(flat_id)
# Otherwise, we miss the @backend afterwards # Otherwise, we miss the @backend afterwards
housing.id = full_flat_id housing.id = full_flat_id
# Eventually clear personal data
if not store_personal_data:
housing.phone = None
return json.dumps(housing, cls=WeboobEncoder) return json.dumps(housing, cls=WeboobEncoder)
except Exception as exc: # pylint: disable=broad-except except Exception as exc: # pylint: disable=broad-except
@ -249,7 +258,8 @@ def fetch_flats(config):
housing_posts = [] housing_posts = []
for query in queries: for query in queries:
housing_posts.extend( housing_posts.extend(
weboob_proxy.query(query, config["max_entries"]) weboob_proxy.query(query, config["max_entries"],
config["store_personal_data"])
) )
LOGGER.info("Fetched %d flats.", len(housing_posts)) LOGGER.info("Fetched %d flats.", len(housing_posts))
@ -270,7 +280,8 @@ def fetch_details(config, flat_id):
""" """
with WeboobProxy(config) as weboob_proxy: with WeboobProxy(config) as weboob_proxy:
LOGGER.info("Loading additional details for flat %s.", flat_id) LOGGER.info("Loading additional details for flat %s.", flat_id)
weboob_output = weboob_proxy.info(flat_id) weboob_output = weboob_proxy.info(flat_id,
config["store_personal_data"])
flat_details = json.loads(weboob_output) flat_details = json.loads(weboob_output)
flat_details = WeboobProxy.restore_decimal_fields(flat_details) flat_details = WeboobProxy.restore_decimal_fields(flat_details)