From aaa9b19cd30a07f4356296760b37822a46d50de9 Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Tue, 13 Nov 2018 11:23:55 +0100 Subject: [PATCH] Rework SQLite database handling to make the script more efficient --- velib.py | 159 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 54 deletions(-) diff --git a/velib.py b/velib.py index 6830b39..f633027 100755 --- a/velib.py +++ b/velib.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 import datetime import json +import logging import os import pybikes import sqlite3 import time +level = logging.WARNING +if os.environ.get('DEBUG', False): + level = logging.DEBUG +logging.basicConfig(level=level) + def db_init(db_name=None): """ @@ -13,6 +19,7 @@ def db_init(db_name=None): Returns a new connection. """ + logging.info('Initialize DB...') now = datetime.datetime.now() if db_name is None: db_name = "week_%s.db" % now.strftime("%V") @@ -48,10 +55,10 @@ def db_init(db_name=None): "timestamp INTEGER, " + "event TEXT, " + "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)") - c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)"); + c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)") + c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)") + c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)") + c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)") conn.commit() return conn @@ -62,17 +69,35 @@ def update_stations(conn): :param conn: Database connection. """ + logging.info('Get all stations from database...') c = conn.cursor() - database_stations = {i[0]: i - for i in - c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()} + database_stations = { + i[0]: i + for i in c.execute( + "SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations" + ).fetchall() + } + logging.info('Get updated Velib stations from the API...') velib = pybikes.get("velib") velib.update() - for station in velib.stations: + fetched_stations = { + station.extra['uid']: station + for station in velib.stations + if 'uid' in station.extra + } + + # List of SQL queries to perform for + events = [] # events happening on stations (temporary closure etc) + stations_update = [] # Update of stations (such as new stands number) + new_stations = [] # New stations to add to the list + stats = [] # Current stats of the station + + logging.info('Processing fetched stations...') + for uid, station in fetched_stations.items(): try: - # Get old station entry if it exists - old_station = database_stations[station.extra["uid"]] + old_station = database_stations[uid] + # Diff the two stations event = [] if station.name != old_station[1]: @@ -87,60 +112,86 @@ def update_stations(conn): event.append({"key": "longitude", "old_value": old_station[4], "new_value": station.longitude}) - if station.extra["banking"] != old_station[5]: + if ( + "banking" in station.extra + and station.extra["banking"] != old_station[5] + ): event.append({"key": "banking", "old_value": old_station[5], "new_value": station.extra["banking"]}) - if station.extra["slots"] != old_station[7]: + if ( + "slots" in station.extra + and station.extra["slots"] != old_station[7] + ): event.append({"key": "bike_stands", "old_value": old_station[7], "new_value": station.extra["slots"]}) # If diff was found if len(event) > 0: - # Update - c.execute("UPDATE " + - "stations " + - "SET name=?, latitude=?, longitude=?, " + - "banking=?, bike_stands=? WHERE id=?", - (station.name, - station.latitude, - station.longitude, - station.extra["banking"], - station.extra["slots"], - station.extra["uid"])) - # And insert event in the table - c.execute("INSERT INTO " + - "stationsevents(station_id, timestamp, event) " + - "VALUES(?, ?, ?)", - (station.extra["uid"], - int(time.time()), - json.dumps(event))) + stations_update.append( + (station.name, + station.latitude, + station.longitude, + station.extra["banking"], + station.extra["slots"], + uid) + ) + events.append( + (uid, int(time.time()), json.dumps(event)) + ) except KeyError: - c.execute("INSERT INTO " + - "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?)", - (station.extra["uid"], - station.name, - "", # Not available - station.latitude, - station.longitude, - station.extra["banking"], - False, # Not available - station.extra["slots"])) - except TypeError: - conn.rollback() - return + # Add the station to the database + new_stations.append( + (uid, + station.name, + "", # Not available + station.latitude, + station.longitude, + station.extra["banking"], + False, # Not available + station.extra["slots"]) + ) + stats.append( + (uid, + station.bikes - station.extra["ebikes"], + station.extra["ebikes"], + station.free, + station.extra["status"], + int(time.time())) # Not available, using current timestamp + ) - c.execute("INSERT INTO " + - "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " + - "VALUES(?, ?, ?, ?, ?, ?)", - (station.extra["uid"], - station.bikes - station.extra["ebikes"], - station.extra["ebikes"], - station.free, - station.extra["status"], - int(time.time()))) # Not available, using current timestamp - conn.commit() + # Update stations + logging.info('Updating stations in db...') + c.executemany( + "UPDATE stations SET name=?, latitude=?, longitude=?, " + + "banking=?, bike_stands=? WHERE id=?", + stations_update + ) + # Insert events in the table + logging.info('Storing stations events in db...') + c.executemany( + "INSERT INTO stationsevents(station_id, timestamp, event) " + + "VALUES(?, ?, ?)", + events + ) + # Add the station to the database + logging.info('Storing new stations in db...') + c.executemany( + "INSERT INTO " + + "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?)", + new_stations + ) + + # Insert the current state in the db + logging.info('Storing current stations stats in db...') + c.executemany( + "INSERT INTO " + + "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " + + "VALUES(?, ?, ?, ?, ?, ?)", + stats + ) + conn.commit() def main():