Optimize SQL handling

velib.py

@@ -1,11 +1,22 @@
 #!/usr/bin/env python3
 import datetime
 import json
+import logging
 import os
 import requests
 import sqlite3
 import time
 
+# Constants
+VELIB_STATIONS_INFOS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json'
+VELIB_STATIONS_STATUS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json'
+
+# Set up logging
+level = logging.WARNING
+if os.environ.get('DEBUG', False):
+    level = logging.DEBUG
+logging.basicConfig(level=level)
+
 
 def db_init(db_name=None):
     """
@@ -26,32 +37,53 @@ def db_init(db_name=None):
     conn = sqlite3.connect(os.path.join(db_folder, db_name))
     c = conn.cursor()
     # Init tables
-    c.execute("CREATE TABLE IF NOT EXISTS stations(" +
+    c.execute("""
-              "id INTEGER, " +
+        CREATE TABLE IF NOT EXISTS stations(
-              "name TEXT, " +
+          id INTEGER,
-              "address TEXT, " +
+          name TEXT,
-              "latitude REAL, " +
+          address TEXT,
-              "longitude REAL, " +
+          latitude REAL,
-              "banking INTEGER, " +
+          longitude REAL,
-              "bonus INTEGER, " +
+          banking INTEGER,
-              "bike_stands INTEGER)")
+          bonus INTEGER,
-    c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" +
+          bike_stands INTEGER
-              "station_id INTEGER, " +
+        )
-              "available_bikes INTEGER, " +
+    """)
-              "available_ebikes INTEGER, " +
+    c.execute("""
-              "free_stands INTEGER, " +
+        CREATE TABLE IF NOT EXISTS stationsstats(
-              "status TEXT, " +
+          station_id INTEGER,
-              "updated INTEGER, " +
+          available_bikes INTEGER,
-              "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)")
+          available_ebikes INTEGER,
-    c.execute("CREATE TABLE IF NOT EXISTS stationsevents(" +
+          free_stands INTEGER,
-              "station_id INTEGER, " +
+          status TEXT,
-              "timestamp INTEGER, " +
+          updated INTEGER,
-              "event TEXT, " +
+          FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
-              "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)")
+        )
-    c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)");
+    """)
-    c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)");
+    c.execute("""
-    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)");
+        CREATE TABLE IF NOT EXISTS stationsevents(
-    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)");
+          station_id INTEGER,
+          timestamp INTEGER,
+          event TEXT,
+          FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
+        )
+    """)
+    c.execute("""
+        CREATE INDEX IF NOT EXISTS
+          stationstats_station_id ON stationsstats (station_id)
+    """)
+    c.execute("""
+        CREATE INDEX IF NOT EXISTS
+          stationsstats_updated ON stationsstats (updated)
+    """)
+    c.execute("""
+        CREATE INDEX IF NOT EXISTS
+          stationsevents_station_id ON stationsevents (station_id)
+    """)
+    c.execute("""
+        CREATE INDEX IF NOT EXISTS
+          stationsevents_timestamp ON stationsevents (timestamp)
+    """)
     conn.commit()
     return conn
 
@@ -63,16 +95,39 @@ def update_stations(conn):
     :param conn: Database connection.
     """
     c = conn.cursor()
-    database_stations = {i[0]: i
-                         for i in
-                         c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()}
 
-    req_stations = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json')
+    logging.info('Get all stations from database...')
+    database_stations = {
+        i[0]: i
+        for i in c.execute("""
+            SELECT
+              id,
+              name,
+              address,
+              latitude,
+              longitude,
+              banking,
+              bonus,
+              bike_stands
+            FROM stations
+        """).fetchall()
+    }
+
+    logging.info('Get updated Velib stations from API...')
+    req_stations = requests.get(VELIB_STATIONS_INFOS_URL)
     stations = {
         station['stationCode']: station
         for station in req_stations.json()['data']['stations']
     }
-    req_status = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json')
+    req_status = requests.get(VELIB_STATIONS_STATUS_URL)
+
+    # List of SQL queries to perform for
+    events = []  # events happening on stations (temporary closure etc)
+    stations_update = []  # Update of stations (such as new stands number)
+    new_stations = []  # New stations to add to the list
+    stats = []  # Current stats of the station
+
+    logging.info('Processing fetched stations')
     for station in req_status.json()['data']['stations']:
         uid = station["stationCode"]
         try:
@@ -98,40 +153,39 @@ def update_stations(conn):
                               "new_value": stations[uid]["capacity"]})
             # If diff was found
             if len(event) > 0:
-                # Update
+                stations_update.append(
-                c.execute("UPDATE " +
+                    (
-                          "stations " +
+                        stations[uid]['name'],
-                          "SET name=?, latitude=?, longitude=?, " +
-                          "banking=?, bike_stands=? WHERE id=?",
-                          (stations[uid]['name'],
                         stations[uid]['lat'],
                         stations[uid]['lon'],
                         None,
                         stations[uid]['capacity'],
-                           uid))
+                        uid
-                # And insert event in the table
+                    )
-                c.execute("INSERT INTO " +
+                )
-                          "stationsevents(station_id, timestamp, event) " +
+                events.append(
-                          "VALUES(?, ?, ?)",
+                    (
-                          (uid,
+                        uid,
                         int(time.time()),
-                           json.dumps(event)))
+                        json.dumps(event)
+                    )
+                )
         except KeyError:
-            c.execute("INSERT INTO " +
+            # Station is unknown, add it
-                      "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " +
+            new_stations.append(
-                      "VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
+                (
-                      (uid,
+                    uid,
                     stations[uid]['name'],
                     "",  # Not available
                     stations[uid]['lat'],
                     stations[uid]['lon'],
                     None,  # Not available
                     False,  # Not available
-                       stations[uid]["capacity"]))
+                    stations[uid]["capacity"]
-        except TypeError:
+                )
-            conn.rollback()
+            )
-            return
 
+        # Add stats
         numEBikesAvailable = (
             station['numBikesAvailable']
             - next(
@@ -140,15 +194,81 @@ def update_stations(conn):
                 if 'ebike' in x
             )
         )
-        c.execute("INSERT INTO " +
+        stats.append(
-                  "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " +
+            (
-                  "VALUES(?, ?, ?, ?, ?, ?)",
+                uid,
-                  (uid,
                 station['numBikesAvailable'],
                 numEBikesAvailable,
                 station['numDocksAvailable'],
                 None,
-                   int(time.time())))  # Not available, using current timestamp
+                int(time.time())  # Not available, using current timestamp
+            )
+        )
+
+    # Update stations
+    logging.info('Updating stations in db...')
+    c.executemany(
+        """
+        UPDATE
+            stations
+        SET
+            name=?, latitude=?, longitude=?,
+            banking=?, bike_stands=?
+        WHERE id=?
+        """,
+        stations_update
+    )
+
+    # Insert events in the table
+    logging.info('Insert stations events in db...')
+    c.executemany(
+        """
+        INSERT INTO
+          stationsevents(station_id, timestamp, event)
+        VALUES(?, ?, ?)
+        """,
+        events
+    )
+
+    # Add the missing stations to database
+    logging.info('Insert missing stations in db...')
+    c.executemany(
+        """
+        INSERT INTO
+        stations(
+          id,
+          name,
+          address,
+          latitude,
+          longitude,
+          banking,
+          bonus,
+          bike_stands
+        )
+        VALUES(?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+        new_stations
+    )
+
+    # Insert stats in database
+    logging.info('Insert stations stats in db...')
+    c.executemany(
+        """
+        INSERT INTO
+        stationsstats(
+          station_id,
+          available_bikes,
+          available_ebikes,
+          free_stands,
+          status,
+          updated
+        )
+        VALUES(?, ?, ?, ?, ?, ?)
+        """,
+        stats
+    )
+
+    # Commit
     conn.commit()