Browse Source

Do not store personal data by default, let user overload this behavior

through config.

Fixes #80.
updated_weboob
Lucas Verney 5 years ago
parent
commit
4a65f52475
  1. 4
      doc/0.getting_started.md
  2. 5
      flatisfy/config.py
  3. 21
      flatisfy/fetch.py

4
doc/0.getting_started.md

@ -113,6 +113,10 @@ List of configuration options: @@ -113,6 +113,10 @@ List of configuration options:
doc](http://bottlepy.org/docs/dev/deployment.html).
* `backends` is a list of Weboob backends to enable. It defaults to any
available and supported Weboob backend.
* `store_personal_data` is a boolean indicated whether or not Flatisfy should
fetch personal data from housing posts and store them in database. Such
personal data include contact phone number for instance. By default,
Flatisfy does not store such personal data.
_Note:_ In production, you can either use the `serve` command with a reliable
webserver instead of the default Bottle webserver (specifying a `webserver`

5
flatisfy/config.py

@ -41,6 +41,9 @@ DEFAULT_CONFIG = { @@ -41,6 +41,9 @@ DEFAULT_CONFIG = {
# Time is in seconds
}
},
# Whether or not to store personal data from housing posts (phone number
# etc)
"store_personal_data": False,
# Navitia API key
"navitia_api_key": None,
# Number of filtering passes to run
@ -133,6 +136,8 @@ def validate_config(config, check_with_data): @@ -133,6 +136,8 @@ def validate_config(config, check_with_data):
assert config["smtp_port"] is None or isinstance(config["smtp_port"], int) # noqa: E501
assert config["smtp_to"] is None or isinstance(config["smtp_to"], list)
assert isinstance(config["store_personal_data"], bool)
# Ensure constraints are ok
assert config["constraints"]
for constraint in config["constraints"].values():

21
flatisfy/fetch.py

@ -173,12 +173,14 @@ class WeboobProxy(object): @@ -173,12 +173,14 @@ class WeboobProxy(object):
return queries
def query(self, query, max_entries=None):
def query(self, query, max_entries=None, store_personal_data=False):
"""
Fetch the housings posts matching a given Weboob query.
:param query: A Weboob `weboob.capabilities.housing.Query`` object.
:param max_entries: Maximum number of entries to fetch.
:param store_personal_data: Whether personal data should be fetched
from housing posts (phone number etc).
:return: The matching housing posts, dumped as a list of JSON objects.
"""
housings = []
@ -188,6 +190,8 @@ class WeboobProxy(object): @@ -188,6 +190,8 @@ class WeboobProxy(object):
self.webnip.do('search_housings', query),
max_entries
):
if not store_personal_data:
housing.phone = None
housings.append(json.dumps(housing, cls=WeboobEncoder))
except CallErrors as exc:
# If an error occured, just log it
@ -197,12 +201,14 @@ class WeboobProxy(object): @@ -197,12 +201,14 @@ class WeboobProxy(object):
)
return housings
def info(self, full_flat_id):
def info(self, full_flat_id, store_personal_data=False):
"""
Get information (details) about an housing post.
:param full_flat_id: A Weboob housing post id, in complete form
(ID@BACKEND)
:param store_personal_data: Whether personal data should be fetched
from housing posts (phone number etc).
:return: The details in JSON.
"""
flat_id, backend_name = full_flat_id.rsplit("@", 1)
@ -220,9 +226,12 @@ class WeboobProxy(object): @@ -220,9 +226,12 @@ class WeboobProxy(object):
housing = backend.get_housing(flat_id)
# Otherwise, we miss the @backend afterwards
housing.id = full_flat_id
# Eventually clear personal data
if not store_personal_data:
housing.phone = None
return json.dumps(housing, cls=WeboobEncoder)
except Exception as exc: # pylint: disable=broad-except
except Exception as exc: # pylint: disable=broad-except
# If an error occured, just log it
LOGGER.error(
"An error occured while fetching housing %s: %s",
@ -249,7 +258,8 @@ def fetch_flats(config): @@ -249,7 +258,8 @@ def fetch_flats(config):
housing_posts = []
for query in queries:
housing_posts.extend(
weboob_proxy.query(query, config["max_entries"])
weboob_proxy.query(query, config["max_entries"],
config["store_personal_data"])
)
LOGGER.info("Fetched %d flats.", len(housing_posts))
@ -270,7 +280,8 @@ def fetch_details(config, flat_id): @@ -270,7 +280,8 @@ def fetch_details(config, flat_id):
"""
with WeboobProxy(config) as weboob_proxy:
LOGGER.info("Loading additional details for flat %s.", flat_id)
weboob_output = weboob_proxy.info(flat_id)
weboob_output = weboob_proxy.info(flat_id,
config["store_personal_data"])
flat_details = json.loads(weboob_output)
flat_details = WeboobProxy.restore_decimal_fields(flat_details)

Loading…
Cancel
Save