phyks/flatisfy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Reduce logs verbosity

Browse Source
This commit is contained in:
Gautier P 2021-01-26 16:43:51 +01:00
parent 0f2a14b024
commit 582a868a1d
2 changed files with 15 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
flatisfy/cmds.py
View File

@ -23,9 +23,7 @@ import time
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
def filter_flats_list( def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, past_flats=None):
config, constraint_name, flats_list, fetch_details=True, past_flats=None
):
""" """
Filter the available flats list. Then, filter it according to criteria. Filter the available flats list. Then, filter it according to criteria.
@ -69,7 +67,7 @@ def filter_flats_list(
use_cache = past_ids.get(flat["id"]) use_cache = past_ids.get(flat["id"])
if use_cache: if use_cache:
LOGGER.info("Skipping details download for %s.", flat["id"]) LOGGER.debug("Skipping details download for %s.", flat["id"])
details = use_cache details = use_cache
else: else:
details = fetch.fetch_details(config, flat["id"]) details = fetch.fetch_details(config, flat["id"])
@ -82,32 +80,22 @@ def filter_flats_list(
# Do a second pass to consolidate all the infos we found and make use of # Do a second pass to consolidate all the infos we found and make use of
# additional infos # additional infos
if config["passes"] > 1: if config["passes"] > 1:
second_pass_result = flatisfy.filters.second_pass( second_pass_result = flatisfy.filters.second_pass(first_pass_result["new"], constraint, config)
first_pass_result["new"], constraint, config
)
else: else:
second_pass_result["new"] = first_pass_result["new"] second_pass_result["new"] = first_pass_result["new"]
# Do a third pass to deduplicate better # Do a third pass to deduplicate better
if config["passes"] > 2: if config["passes"] > 2:
third_pass_result = flatisfy.filters.third_pass( third_pass_result = flatisfy.filters.third_pass(second_pass_result["new"], config)
second_pass_result["new"], config
)
else: else:
third_pass_result["new"] = second_pass_result["new"] third_pass_result["new"] = second_pass_result["new"]
return { return {
"new": third_pass_result["new"], "new": third_pass_result["new"],
"duplicate": ( "duplicate": (
first_pass_result["duplicate"] first_pass_result["duplicate"] + second_pass_result["duplicate"] + third_pass_result["duplicate"]
+ second_pass_result["duplicate"]
+ third_pass_result["duplicate"]
),
"ignored": (
first_pass_result["ignored"]
+ second_pass_result["ignored"]
+ third_pass_result["ignored"]
), ),
"ignored": (first_pass_result["ignored"] + second_pass_result["ignored"] + third_pass_result["ignored"]),
} }
@ -178,10 +166,7 @@ def import_and_filter(config, load_from_db=False, new_only=False):
for status, flats_list in flatten_flats_by_status.items(): for status, flats_list in flatten_flats_by_status.items():
# Build SQLAlchemy Flat model objects for every available flat # Build SQLAlchemy Flat model objects for every available flat
flats_objects = { flats_objects = {flat_dict["id"]: flat_model.Flat.from_dict(flat_dict) for flat_dict in flats_list}
flat_dict["id"]: flat_model.Flat.from_dict(flat_dict)
for flat_dict in flats_list
}
if flats_objects: if flats_objects:
# If there are some flats, try to merge them with the ones in # If there are some flats, try to merge them with the ones in
@ -266,8 +251,5 @@ def serve(config):
# standard logging # standard logging
server = web_app.QuietWSGIRefServer server = web_app.QuietWSGIRefServer
print( print("Launching web viewer running on http://%s:%s" % (config["host"], config["port"]))
"Launching web viewer running on http://%s:%s"
% (config["host"], config["port"])
)
app.run(host=config["host"], port=config["port"], server=server) app.run(host=config["host"], port=config["port"], server=server)

24
flatisfy/filters/__init__.py
View File

@ -37,15 +37,13 @@ def refine_with_housing_criteria(flats_list, constraint):
# Check postal code # Check postal code
postal_code = flat["flatisfy"].get("postal_code", None) postal_code = flat["flatisfy"].get("postal_code", None)
if postal_code and postal_code not in constraint["postal_codes"]: if postal_code and postal_code not in constraint["postal_codes"]:
LOGGER.info("Postal code for flat %s is out of range.", flat["id"]) LOGGER.info("Postal code %s for flat %s is out of range.", postal_code, flat["id"])
is_ok[i] = is_ok[i] and False is_ok[i] = is_ok[i] and False
# Check time_to # Check time_to
for place_name, time in flat["flatisfy"].get("time_to", {}).items(): for place_name, time in flat["flatisfy"].get("time_to", {}).items():
time = time["time"] time = time["time"]
is_within_interval = tools.is_within_interval( is_within_interval = tools.is_within_interval(time, *(constraint["time_to"][place_name]["time"]))
time, *(constraint["time_to"][place_name]["time"])
)
if not is_within_interval: if not is_within_interval:
LOGGER.info( LOGGER.info(
"Flat %s is too far from place %s: %ds.", "Flat %s is too far from place %s: %ds.",
@ -58,12 +56,10 @@ def refine_with_housing_criteria(flats_list, constraint):
# Check other fields # Check other fields
for field in ["area", "cost", "rooms", "bedrooms"]: for field in ["area", "cost", "rooms", "bedrooms"]:
interval = constraint[field] interval = constraint[field]
is_within_interval = tools.is_within_interval( is_within_interval = tools.is_within_interval(flat.get(field, None), *interval)
flat.get(field, None), *interval
)
if not is_within_interval: if not is_within_interval:
LOGGER.info( LOGGER.info(
"%s for flat %s is out of range.", field.capitalize(), flat["id"] "%s %s for flat %s is out of range.", field.capitalize(), str(flat.get(field, None)), flat["id"]
) )
is_ok[i] = is_ok[i] and is_within_interval is_ok[i] = is_ok[i] and is_within_interval
@ -95,9 +91,7 @@ def refine_with_details_criteria(flats_list, constraint):
for i, flat in enumerate(flats_list): for i, flat in enumerate(flats_list):
# Check number of pictures # Check number of pictures
has_enough_photos = tools.is_within_interval( has_enough_photos = tools.is_within_interval(len(flat.get("photos", [])), constraint["minimum_nb_photos"], None)
len(flat.get("photos", [])), constraint["minimum_nb_photos"], None
)
if not has_enough_photos: if not has_enough_photos:
LOGGER.info( LOGGER.info(
"Flat %s only has %d photos, it should have at least %d.", "Flat %s only has %d photos, it should have at least %d.",
@ -148,16 +142,12 @@ def first_pass(flats_list, constraint, config):
# Handle duplicates based on ids # Handle duplicates based on ids
# Just remove them (no merge) as they should be the exact same object. # Just remove them (no merge) as they should be the exact same object.
flats_list, _ = duplicates.detect( flats_list, _ = duplicates.detect(flats_list, key="id", merge=False, should_intersect=False)
flats_list, key="id", merge=False, should_intersect=False
)
# Also merge duplicates based on urls (these may come from different # Also merge duplicates based on urls (these may come from different
# flatboob backends) # flatboob backends)
# This is especially useful as some websites such as entreparticuliers # This is especially useful as some websites such as entreparticuliers
# contains a lot of leboncoin housings posts. # contains a lot of leboncoin housings posts.
flats_list, duplicates_by_urls = duplicates.detect( flats_list, duplicates_by_urls = duplicates.detect(flats_list, key="urls", merge=True, should_intersect=True)
flats_list, key="urls", merge=True, should_intersect=True
)
# Guess the postal codes # Guess the postal codes
flats_list = metadata.guess_postal_code(flats_list, constraint, config) flats_list = metadata.guess_postal_code(flats_list, constraint, config)