From a5aeadef86744639e6755f32e607065ae86ce9af Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Thu, 4 May 2017 20:52:10 +0200 Subject: [PATCH] Various bug fixes Fix some bugs introduced in recent developments as well as some UI issues (and i18n). Closes issue #42. --- flatisfy/cmds.py | 18 ++++++++++++++--- flatisfy/config.py | 4 ++-- flatisfy/filters/__init__.py | 29 ++++++++++++++++++++++----- flatisfy/filters/duplicates.py | 1 + flatisfy/web/js_src/views/details.vue | 4 ++++ flatisfy/web/js_src/views/home.vue | 2 +- flatisfy/web/js_src/views/search.vue | 4 ++-- flatisfy/web/js_src/views/status.vue | 2 +- flatisfy/web/routes/api.py | 2 +- 9 files changed, 51 insertions(+), 15 deletions(-) diff --git a/flatisfy/cmds.py b/flatisfy/cmds.py index ece9a31..2108fed 100644 --- a/flatisfy/cmds.py +++ b/flatisfy/cmds.py @@ -35,6 +35,7 @@ def filter_flats(config, flats_list, fetch_details=True): first_pass_result = collections.defaultdict(list) second_pass_result = collections.defaultdict(list) + third_pass_result = collections.defaultdict(list) # Do a first pass with the available infos to try to remove as much # unwanted postings as possible if config["passes"] > 0: @@ -58,14 +59,25 @@ def filter_flats(config, flats_list, fetch_details=True): else: second_pass_result["new"] = first_pass_result["new"] + # Do a third pass to deduplicate better + if config["passes"] > 2: + third_pass_result = flatisfy.filters.third_pass( + second_pass_result["new"], config + ) + else: + third_pass_result["new"] = second_pass_result["new"] + return { - "new": second_pass_result["new"], + "new": third_pass_result["new"], "duplicate": ( first_pass_result["duplicate"] + - second_pass_result["duplicate"] + second_pass_result["duplicate"] + + third_pass_result["duplicate"] ), "ignored": ( - first_pass_result["ignored"] + second_pass_result["ignored"] + first_pass_result["ignored"] + + second_pass_result["ignored"] + + third_pass_result["ignored"] ) } diff --git a/flatisfy/config.py b/flatisfy/config.py index ff4b8bb..55ad87f 100644 --- a/flatisfy/config.py +++ b/flatisfy/config.py @@ -38,7 +38,7 @@ DEFAULT_CONFIG = { # Navitia API key "navitia_api_key": None, # Number of filtering passes to run - "passes": 2, + "passes": 3, # Maximum number of entries to fetch "max_entries": None, # Directory in wich data will be put. ``None`` is XDG default location. @@ -129,7 +129,7 @@ def validate_config(config): assert "time" in item _check_constraints_bounds(item["time"]) - assert config["passes"] in [0, 1, 2] + assert config["passes"] in [0, 1, 2, 3] assert config["max_entries"] is None or (isinstance(config["max_entries"], int) and config["max_entries"] > 0) # noqa: E501 assert config["data_directory"] is None or isinstance(config["data_directory"], str) # noqa: E501 diff --git a/flatisfy/filters/__init__.py b/flatisfy/filters/__init__.py index 675a5cd..17f77ea 100644 --- a/flatisfy/filters/__init__.py +++ b/flatisfy/filters/__init__.py @@ -44,13 +44,14 @@ def refine_with_housing_criteria(flats_list, config): # Check time_to for place_name, time in flat["flatisfy"].get("time_to", {}).items(): + time = time["time"] is_within_interval = tools.is_within_interval( time, *(config["constraints"]["time_to"][place_name]["time"]) ) if not is_within_interval: - LOGGER.info("Flat %s is too far from place %s.", - flat["id"], place_name) + LOGGER.info("Flat %s is too far from place %s: %ds.", + flat["id"], place_name, time) is_ok[i] = is_ok[i] and is_within_interval # Check other fields @@ -148,14 +149,32 @@ def second_pass(flats_list, config): # Compute travel time to specified points flats_list = metadata.compute_travel_times(flats_list, config) - # Deduplicate the list using every available data - flats_list, duplicate_flats = duplicates.deep_detect(flats_list) - # Remove returned housing posts that do not match criteria flats_list, ignored_list = refine_with_housing_criteria(flats_list, config) return { "new": flats_list, "ignored": ignored_list, + "duplicate": [] + } + + +def third_pass(flats_list, config): + """ + Third filtering pass. + + This pass is expected to perform deep duplicate detection on available + flats. + + :param flats_list: A list of flats dict to filter. + :param config: A config dict. + :return: A dict mapping flat status and list of flat objects. + """ + # Deduplicate the list using every available data + flats_list, duplicate_flats = duplicates.deep_detect(flats_list) + + return { + "new": flats_list, + "ignored": [], "duplicate": duplicate_flats } diff --git a/flatisfy/filters/duplicates.py b/flatisfy/filters/duplicates.py index 6c806d6..0240c03 100644 --- a/flatisfy/filters/duplicates.py +++ b/flatisfy/filters/duplicates.py @@ -184,6 +184,7 @@ def deep_detect(flats_list): the flats objects that should be removed and considered as duplicates (they were already merged). """ + LOGGER.info("Running deep duplicates detection.") matching_flats = collections.defaultdict(list) for i, flat1 in enumerate(flats_list): matching_flats[flat1["id"]].append(flat1["id"]) diff --git a/flatisfy/web/js_src/views/details.vue b/flatisfy/web/js_src/views/details.vue index 5af3ce2..25f12ef 100644 --- a/flatisfy/web/js_src/views/details.vue +++ b/flatisfy/web/js_src/views/details.vue @@ -268,6 +268,10 @@ export default { } }, + updateFlatStatus (status) { + this.$store.dispatch('updateFlatStatus', { flatId: this.$route.params.id, newStatus: status }) + }, + updateFlatNotes () { const notes = this.$refs.notesTextarea.value this.$store.dispatch( diff --git a/flatisfy/web/js_src/views/home.vue b/flatisfy/web/js_src/views/home.vue index d6f7a73..4a1e41d 100644 --- a/flatisfy/web/js_src/views/home.vue +++ b/flatisfy/web/js_src/views/home.vue @@ -6,7 +6,7 @@

{{ $t("home.new_available_flats") }}

diff --git a/flatisfy/web/js_src/views/search.vue b/flatisfy/web/js_src/views/search.vue index 23f381b..0860a36 100644 --- a/flatisfy/web/js_src/views/search.vue +++ b/flatisfy/web/js_src/views/search.vue @@ -14,7 +14,7 @@ @@ -47,7 +47,7 @@ export default { } return this.$store.getters.postalCodesFlatsBuckets( - flat => flat.status !== 'duplicate' && flat.status !== 'ignored' + flat => flat.status !== 'duplicate' && flat.status !== 'ignored' && flat.status !== "user_deleted" ) }, loading () { diff --git a/flatisfy/web/js_src/views/status.vue b/flatisfy/web/js_src/views/status.vue index 6218625..6928bcd 100644 --- a/flatisfy/web/js_src/views/status.vue +++ b/flatisfy/web/js_src/views/status.vue @@ -15,7 +15,7 @@ diff --git a/flatisfy/web/routes/api.py b/flatisfy/web/routes/api.py index 78a113c..b2b9a7e 100644 --- a/flatisfy/web/routes/api.py +++ b/flatisfy/web/routes/api.py @@ -191,7 +191,7 @@ def update_flat_notation_v1(flat_id, db): try: flat.notation = json.load(bottle.request.body)["notation"] - assert flat.notes >= 0 and flat.notes <= 5 + assert flat.notation >= 0 and flat.notation <= 5 except (AssertionError, ValueError, KeyError): return bottle.HTTPError(400, "Invalid notation provided.")