From 4d117263325020d6a135f56441f94ba92a65f62d Mon Sep 17 00:00:00 2001
From: Gautier P <gautier_git@damsy.net>
Date: Mon, 8 Feb 2021 16:54:20 +0100
Subject: [PATCH] Use ratelimit to avoid rate bans on some APIs

---
 flatisfy/cmds.py  | 13 +++++++++----
 flatisfy/fetch.py |  9 +++++++++
 requirements.txt  |  1 +
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/flatisfy/cmds.py b/flatisfy/cmds.py
index 8a3dd1f..7216d17 100644
--- a/flatisfy/cmds.py
+++ b/flatisfy/cmds.py
@@ -19,6 +19,7 @@ from flatisfy import tools
 from flatisfy.filters import metadata
 from flatisfy.web import app as web_app
 import time
+from ratelimit.exception import RateLimitException
 
 LOGGER = logging.getLogger(__name__)
 
@@ -70,10 +71,14 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True, p
                 LOGGER.debug("Skipping details download for %s.", flat["id"])
                 details = use_cache
             else:
-                details = fetch.fetch_details(config, flat["id"])
-                if flat["id"].endswith("@leboncoin"):
-                    # sleep 0.5s to avoid rate-kick
-                    time.sleep(0.5)
+                if flat["id"].split("@")[1] in ["seloger", "leboncoin"]:
+                    try:
+                        details = fetch.fetch_details_rate_limited(config, flat["id"])
+                    except RateLimitException:
+                        time.sleep(60)
+                        details = fetch.fetch_details_rate_limited(config, flat["id"])
+                else:
+                    details = fetch.fetch_details(config, flat["id"])
 
             first_pass_result["new"][i] = tools.merge_dicts(flat, details)
 
diff --git a/flatisfy/fetch.py b/flatisfy/fetch.py
index 736aefb..78576d7 100644
--- a/flatisfy/fetch.py
+++ b/flatisfy/fetch.py
@@ -9,6 +9,7 @@ import collections
 import itertools
 import json
 import logging
+from ratelimit import limits
 
 from flatisfy import database
 from flatisfy import tools
@@ -248,6 +249,14 @@ def fetch_flats(config):
     return fetched_flats
 
 
+@limits(calls=10, period=60)
+def fetch_details_rate_limited(config, flat_id):
+    """
+    Limit flats fetching to at most 10 calls per minute to avoid rate banning
+    """
+    return fetch_details(config, flat_id)
+
+
 def fetch_details(config, flat_id):
     """
     Fetch the additional details for a flat using Flatboob / WebOOB.
diff --git a/requirements.txt b/requirements.txt
index 374afb9..b7e6121 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,6 +8,7 @@ future
 imagehash
 mapbox
 pillow
+ratelimit
 requests
 requests_mock
 sqlalchemy