Use ratelimit to avoid rate bans on some APIs

flatisfy/cmds.py

@@ -19,6 +19,7 @@ from flatisfy import tools
 from flatisfy.filters import metadata
 from flatisfy.web import app as web_app
 import time
+from ratelimit.exception import RateLimitException
 
 LOGGER = logging.getLogger(__name__)
 
@@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p
                 LOGGER.debug("Skipping details download for %s.", flat["id"])
                 details = use_cache
             else:
-                details = fetch.fetch_details(config, flat["id"])
+                if flat["id"].split("@")[1] in ["seloger", "leboncoin"]:
-                if flat["id"].endswith("@leboncoin"):
+                    try:
-                    # sleep 0.5s to avoid rate-kick
+                        details = fetch.fetch_details_rate_limited(config, flat["id"])
-                    time.sleep(0.5)
+                    except RateLimitException:
+                        time.sleep(60)
+                        details = fetch.fetch_details_rate_limited(config, flat["id"])
+                else:
+                    details = fetch.fetch_details(config, flat["id"])
 
             first_pass_result["new"][i] = tools.merge_dicts(flat, details)
 

flatisfy/fetch.py

@@ -9,6 +9,7 @@ import collections
 import itertools
 import json
 import logging
+from ratelimit import limits
 
 from flatisfy import database
 from flatisfy import tools
@@ -248,6 +249,14 @@ def fetch_flats(config):
     return fetched_flats
 
 
+@limits(calls=10, period=60)
+def fetch_details_rate_limited(config, flat_id):
+    """
+    Limit flats fetching to at most 10 calls per minute to avoid rate banning
+    """
+    return fetch_details(config, flat_id)
+
+
 def fetch_details(config, flat_id):
     """
     Fetch the additional details for a flat using Flatboob / WebOOB.

requirements.txt

@@ -8,6 +8,7 @@ future
 imagehash
 mapbox
 pillow
+ratelimit
 requests
 requests_mock
 sqlalchemy