From 47ce1ba7e07f7e33b12b81e3b8c646dd0cf50b2f Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Wed, 10 Jan 2018 16:46:47 +0100 Subject: [PATCH] Ignore duplicates by IDs, no need to even add the duplicate in db --- flatisfy/filters/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flatisfy/filters/__init__.py b/flatisfy/filters/__init__.py index 15a15ce..4efadac 100644 --- a/flatisfy/filters/__init__.py +++ b/flatisfy/filters/__init__.py @@ -162,7 +162,7 @@ def first_pass(flats_list, constraint, config): # Handle duplicates based on ids # Just remove them (no merge) as they should be the exact same object. - flats_list, duplicates_by_id = duplicates.detect( + flats_list, _ = duplicates.detect( flats_list, key="id", merge=False, should_intersect=False ) # Also merge duplicates based on urls (these may come from different @@ -184,7 +184,7 @@ def first_pass(flats_list, constraint, config): return { "new": flats_list, "ignored": ignored_list, - "duplicate": duplicates_by_id + duplicates_by_urls + "duplicate": duplicates_by_urls } @tools.timeit