diff --git a/doc/0.getting_started.md b/doc/0.getting_started.md index f607bae..a2f7c2c 100644 --- a/doc/0.getting_started.md +++ b/doc/0.getting_started.md @@ -113,6 +113,10 @@ List of configuration options: doc](http://bottlepy.org/docs/dev/deployment.html). * `backends` is a list of Weboob backends to enable. It defaults to any available and supported Weboob backend. +* `store_personal_data` is a boolean indicated whether or not Flatisfy should + fetch personal data from housing posts and store them in database. Such + personal data include contact phone number for instance. By default, + Flatisfy does not store such personal data. _Note:_ In production, you can either use the `serve` command with a reliable webserver instead of the default Bottle webserver (specifying a `webserver` diff --git a/flatisfy/config.py b/flatisfy/config.py index 9d6171f..669e453 100644 --- a/flatisfy/config.py +++ b/flatisfy/config.py @@ -41,6 +41,9 @@ DEFAULT_CONFIG = { # Time is in seconds } }, + # Whether or not to store personal data from housing posts (phone number + # etc) + "store_personal_data": False, # Navitia API key "navitia_api_key": None, # Number of filtering passes to run @@ -133,6 +136,8 @@ def validate_config(config, check_with_data): assert config["smtp_port"] is None or isinstance(config["smtp_port"], int) # noqa: E501 assert config["smtp_to"] is None or isinstance(config["smtp_to"], list) + assert isinstance(config["store_personal_data"], bool) + # Ensure constraints are ok assert config["constraints"] for constraint in config["constraints"].values(): diff --git a/flatisfy/fetch.py b/flatisfy/fetch.py index 15f4186..f35a52e 100644 --- a/flatisfy/fetch.py +++ b/flatisfy/fetch.py @@ -173,12 +173,14 @@ class WeboobProxy(object): return queries - def query(self, query, max_entries=None): + def query(self, query, max_entries=None, store_personal_data=False): """ Fetch the housings posts matching a given Weboob query. :param query: A Weboob `weboob.capabilities.housing.Query`` object. :param max_entries: Maximum number of entries to fetch. + :param store_personal_data: Whether personal data should be fetched + from housing posts (phone number etc). :return: The matching housing posts, dumped as a list of JSON objects. """ housings = [] @@ -188,6 +190,8 @@ class WeboobProxy(object): self.webnip.do('search_housings', query), max_entries ): + if not store_personal_data: + housing.phone = None housings.append(json.dumps(housing, cls=WeboobEncoder)) except CallErrors as exc: # If an error occured, just log it @@ -197,12 +201,14 @@ class WeboobProxy(object): ) return housings - def info(self, full_flat_id): + def info(self, full_flat_id, store_personal_data=False): """ Get information (details) about an housing post. :param full_flat_id: A Weboob housing post id, in complete form (ID@BACKEND) + :param store_personal_data: Whether personal data should be fetched + from housing posts (phone number etc). :return: The details in JSON. """ flat_id, backend_name = full_flat_id.rsplit("@", 1) @@ -220,9 +226,12 @@ class WeboobProxy(object): housing = backend.get_housing(flat_id) # Otherwise, we miss the @backend afterwards housing.id = full_flat_id + # Eventually clear personal data + if not store_personal_data: + housing.phone = None return json.dumps(housing, cls=WeboobEncoder) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except # If an error occured, just log it LOGGER.error( "An error occured while fetching housing %s: %s", @@ -249,7 +258,8 @@ def fetch_flats(config): housing_posts = [] for query in queries: housing_posts.extend( - weboob_proxy.query(query, config["max_entries"]) + weboob_proxy.query(query, config["max_entries"], + config["store_personal_data"]) ) LOGGER.info("Fetched %d flats.", len(housing_posts)) @@ -270,7 +280,8 @@ def fetch_details(config, flat_id): """ with WeboobProxy(config) as weboob_proxy: LOGGER.info("Loading additional details for flat %s.", flat_id) - weboob_output = weboob_proxy.info(flat_id) + weboob_output = weboob_proxy.info(flat_id, + config["store_personal_data"]) flat_details = json.loads(weboob_output) flat_details = WeboobProxy.restore_decimal_fields(flat_details)