Use ratelimit to avoid rate bans on some APIs

This commit is contained in:
Gautier P 2021-02-08 16:54:20 +01:00
parent 1bd855dbd8
commit 4d11726332
3 changed files with 19 additions and 4 deletions

View File

@ -19,6 +19,7 @@ from flatisfy import tools
from flatisfy.filters import metadata from flatisfy.filters import metadata
from flatisfy.web import app as web_app from flatisfy.web import app as web_app
import time import time
from ratelimit.exception import RateLimitException
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p
LOGGER.debug("Skipping details download for %s.", flat["id"]) LOGGER.debug("Skipping details download for %s.", flat["id"])
details = use_cache details = use_cache
else: else:
details = fetch.fetch_details(config, flat["id"]) if flat["id"].split("@")[1] in ["seloger", "leboncoin"]:
if flat["id"].endswith("@leboncoin"): try:
# sleep 0.5s to avoid rate-kick details = fetch.fetch_details_rate_limited(config, flat["id"])
time.sleep(0.5) except RateLimitException:
time.sleep(60)
details = fetch.fetch_details_rate_limited(config, flat["id"])
else:
details = fetch.fetch_details(config, flat["id"])
first_pass_result["new"][i] = tools.merge_dicts(flat, details) first_pass_result["new"][i] = tools.merge_dicts(flat, details)

View File

@ -9,6 +9,7 @@ import collections
import itertools import itertools
import json import json
import logging import logging
from ratelimit import limits
from flatisfy import database from flatisfy import database
from flatisfy import tools from flatisfy import tools
@ -248,6 +249,14 @@ def fetch_flats(config):
return fetched_flats return fetched_flats
@limits(calls=10, period=60)
def fetch_details_rate_limited(config, flat_id):
"""
Limit flats fetching to at most 10 calls per minute to avoid rate banning
"""
return fetch_details(config, flat_id)
def fetch_details(config, flat_id): def fetch_details(config, flat_id):
""" """
Fetch the additional details for a flat using Flatboob / WebOOB. Fetch the additional details for a flat using Flatboob / WebOOB.

View File

@ -8,6 +8,7 @@ future
imagehash imagehash
mapbox mapbox
pillow pillow
ratelimit
requests requests
requests_mock requests_mock
sqlalchemy sqlalchemy