Browse Source

Use ratelimit to avoid rate bans on some APIs

master
Gautier P 1 year ago
parent
commit
4d11726332
  1. 13
      flatisfy/cmds.py
  2. 9
      flatisfy/fetch.py
  3. 1
      requirements.txt

13
flatisfy/cmds.py

@ -19,6 +19,7 @@ from flatisfy import tools @@ -19,6 +19,7 @@ from flatisfy import tools
from flatisfy.filters import metadata
from flatisfy.web import app as web_app
import time
from ratelimit.exception import RateLimitException
LOGGER = logging.getLogger(__name__)
@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p @@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p
LOGGER.debug("Skipping details download for %s.", flat["id"])
details = use_cache
else:
details = fetch.fetch_details(config, flat["id"])
if flat["id"].endswith("@leboncoin"):
# sleep 0.5s to avoid rate-kick
time.sleep(0.5)
if flat["id"].split("@")[1] in ["seloger", "leboncoin"]:
try:
details = fetch.fetch_details_rate_limited(config, flat["id"])
except RateLimitException:
time.sleep(60)
details = fetch.fetch_details_rate_limited(config, flat["id"])
else:
details = fetch.fetch_details(config, flat["id"])
first_pass_result["new"][i] = tools.merge_dicts(flat, details)

9
flatisfy/fetch.py

@ -9,6 +9,7 @@ import collections @@ -9,6 +9,7 @@ import collections
import itertools
import json
import logging
from ratelimit import limits
from flatisfy import database
from flatisfy import tools
@ -248,6 +249,14 @@ def fetch_flats(config): @@ -248,6 +249,14 @@ def fetch_flats(config):
return fetched_flats
@limits(calls=10, period=60)
def fetch_details_rate_limited(config, flat_id):
"""
Limit flats fetching to at most 10 calls per minute to avoid rate banning
"""
return fetch_details(config, flat_id)
def fetch_details(config, flat_id):
"""
Fetch the additional details for a flat using Flatboob / WebOOB.

1
requirements.txt

@ -8,6 +8,7 @@ future @@ -8,6 +8,7 @@ future
imagehash
mapbox
pillow
ratelimit
requests
requests_mock
sqlalchemy

Loading…
Cancel
Save