diff --git a/velib.py b/velib.py index 2ee0121..1530dcc 100755 --- a/velib.py +++ b/velib.py @@ -1,11 +1,22 @@ #!/usr/bin/env python3 import datetime import json +import logging import os import requests import sqlite3 import time +# Constants +VELIB_STATIONS_INFOS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json' +VELIB_STATIONS_STATUS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json' + +# Set up logging +level = logging.WARNING +if os.environ.get('DEBUG', False): + level = logging.DEBUG +logging.basicConfig(level=level) + def db_init(db_name=None): """ @@ -26,32 +37,53 @@ def db_init(db_name=None): conn = sqlite3.connect(os.path.join(db_folder, db_name)) c = conn.cursor() # Init tables - c.execute("CREATE TABLE IF NOT EXISTS stations(" + - "id INTEGER, " + - "name TEXT, " + - "address TEXT, " + - "latitude REAL, " + - "longitude REAL, " + - "banking INTEGER, " + - "bonus INTEGER, " + - "bike_stands INTEGER)") - c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" + - "station_id INTEGER, " + - "available_bikes INTEGER, " + - "available_ebikes INTEGER, " + - "free_stands INTEGER, " + - "status TEXT, " + - "updated INTEGER, " + - "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)") - c.execute("CREATE TABLE IF NOT EXISTS stationsevents(" + - "station_id INTEGER, " + - "timestamp INTEGER, " + - "event TEXT, " + - "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)") - c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)"); - c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)"); + c.execute(""" + CREATE TABLE IF NOT EXISTS stations( + id INTEGER, + name TEXT, + address TEXT, + latitude REAL, + longitude REAL, + banking INTEGER, + bonus INTEGER, + bike_stands INTEGER + ) + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS stationsstats( + station_id INTEGER, + available_bikes INTEGER, + available_ebikes INTEGER, + free_stands INTEGER, + status TEXT, + updated INTEGER, + FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE + ) + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS stationsevents( + station_id INTEGER, + timestamp INTEGER, + event TEXT, + FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE + ) + """) + c.execute(""" + CREATE INDEX IF NOT EXISTS + stationstats_station_id ON stationsstats (station_id) + """) + c.execute(""" + CREATE INDEX IF NOT EXISTS + stationsstats_updated ON stationsstats (updated) + """) + c.execute(""" + CREATE INDEX IF NOT EXISTS + stationsevents_station_id ON stationsevents (station_id) + """) + c.execute(""" + CREATE INDEX IF NOT EXISTS + stationsevents_timestamp ON stationsevents (timestamp) + """) conn.commit() return conn @@ -63,16 +95,39 @@ def update_stations(conn): :param conn: Database connection. """ c = conn.cursor() - database_stations = {i[0]: i - for i in - c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()} - req_stations = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json') + logging.info('Get all stations from database...') + database_stations = { + i[0]: i + for i in c.execute(""" + SELECT + id, + name, + address, + latitude, + longitude, + banking, + bonus, + bike_stands + FROM stations + """).fetchall() + } + + logging.info('Get updated Velib stations from API...') + req_stations = requests.get(VELIB_STATIONS_INFOS_URL) stations = { station['stationCode']: station for station in req_stations.json()['data']['stations'] } - req_status = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json') + req_status = requests.get(VELIB_STATIONS_STATUS_URL) + + # List of SQL queries to perform for + events = [] # events happening on stations (temporary closure etc) + stations_update = [] # Update of stations (such as new stands number) + new_stations = [] # New stations to add to the list + stats = [] # Current stats of the station + + logging.info('Processing fetched stations') for station in req_status.json()['data']['stations']: uid = station["stationCode"] try: @@ -98,40 +153,39 @@ def update_stations(conn): "new_value": stations[uid]["capacity"]}) # If diff was found if len(event) > 0: - # Update - c.execute("UPDATE " + - "stations " + - "SET name=?, latitude=?, longitude=?, " + - "banking=?, bike_stands=? WHERE id=?", - (stations[uid]['name'], - stations[uid]['lat'], - stations[uid]['lon'], - None, - stations[uid]['capacity'], - uid)) - # And insert event in the table - c.execute("INSERT INTO " + - "stationsevents(station_id, timestamp, event) " + - "VALUES(?, ?, ?)", - (uid, - int(time.time()), - json.dumps(event))) + stations_update.append( + ( + stations[uid]['name'], + stations[uid]['lat'], + stations[uid]['lon'], + None, + stations[uid]['capacity'], + uid + ) + ) + events.append( + ( + uid, + int(time.time()), + json.dumps(event) + ) + ) except KeyError: - c.execute("INSERT INTO " + - "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?)", - (uid, - stations[uid]['name'], - "", # Not available - stations[uid]['lat'], - stations[uid]['lon'], - None, # Not available - False, # Not available - stations[uid]["capacity"])) - except TypeError: - conn.rollback() - return + # Station is unknown, add it + new_stations.append( + ( + uid, + stations[uid]['name'], + "", # Not available + stations[uid]['lat'], + stations[uid]['lon'], + None, # Not available + False, # Not available + stations[uid]["capacity"] + ) + ) + # Add stats numEBikesAvailable = ( station['numBikesAvailable'] - next( @@ -140,16 +194,82 @@ def update_stations(conn): if 'ebike' in x ) ) - c.execute("INSERT INTO " + - "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " + - "VALUES(?, ?, ?, ?, ?, ?)", - (uid, - station['numBikesAvailable'], - numEBikesAvailable, - station['numDocksAvailable'], - None, - int(time.time()))) # Not available, using current timestamp - conn.commit() + stats.append( + ( + uid, + station['numBikesAvailable'], + numEBikesAvailable, + station['numDocksAvailable'], + None, + int(time.time()) # Not available, using current timestamp + ) + ) + + # Update stations + logging.info('Updating stations in db...') + c.executemany( + """ + UPDATE + stations + SET + name=?, latitude=?, longitude=?, + banking=?, bike_stands=? + WHERE id=? + """, + stations_update + ) + + # Insert events in the table + logging.info('Insert stations events in db...') + c.executemany( + """ + INSERT INTO + stationsevents(station_id, timestamp, event) + VALUES(?, ?, ?) + """, + events + ) + + # Add the missing stations to database + logging.info('Insert missing stations in db...') + c.executemany( + """ + INSERT INTO + stations( + id, + name, + address, + latitude, + longitude, + banking, + bonus, + bike_stands + ) + VALUES(?, ?, ?, ?, ?, ?, ?, ?) + """, + new_stations + ) + + # Insert stats in database + logging.info('Insert stations stats in db...') + c.executemany( + """ + INSERT INTO + stationsstats( + station_id, + available_bikes, + available_ebikes, + free_stands, + status, + updated + ) + VALUES(?, ?, ?, ?, ?, ?) + """, + stats + ) + + # Commit + conn.commit() def main():