Use ratelimit to avoid rate bans on some APIs

This commit is contained in:
Gautier P 2021-02-08 16:54:20 +01:00
parent 1bd855dbd8
commit 4d11726332
3 changed files with 19 additions and 4 deletions

View File

@ -19,6 +19,7 @@ from flatisfy import tools
from flatisfy.filters import metadata
from flatisfy.web import app as web_app
import time
from ratelimit.exception import RateLimitException
LOGGER = logging.getLogger(__name__)
@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p
LOGGER.debug("Skipping details download for %s.", flat["id"])
details = use_cache
else:
details = fetch.fetch_details(config, flat["id"])
if flat["id"].endswith("@leboncoin"):
# sleep 0.5s to avoid rate-kick
time.sleep(0.5)
if flat["id"].split("@")[1] in ["seloger", "leboncoin"]:
try:
details = fetch.fetch_details_rate_limited(config, flat["id"])
except RateLimitException:
time.sleep(60)
details = fetch.fetch_details_rate_limited(config, flat["id"])
else:
details = fetch.fetch_details(config, flat["id"])
first_pass_result["new"][i] = tools.merge_dicts(flat, details)

View File

@ -9,6 +9,7 @@ import collections
import itertools
import json
import logging
from ratelimit import limits
from flatisfy import database
from flatisfy import tools
@ -248,6 +249,14 @@ def fetch_flats(config):
return fetched_flats
@limits(calls=10, period=60)
def fetch_details_rate_limited(config, flat_id):
"""
Limit flats fetching to at most 10 calls per minute to avoid rate banning
"""
return fetch_details(config, flat_id)
def fetch_details(config, flat_id):
"""
Fetch the additional details for a flat using Flatboob / WebOOB.

View File

@ -8,6 +8,7 @@ future
imagehash
mapbox
pillow
ratelimit
requests
requests_mock
sqlalchemy