phyks
/
velibdataset
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Optimize SQL handling

This commit is contained in:
Lucas Verney 2022-04-09 12:02:05 +02:00
parent 79c0dc3b29
commit f3894c7c9b
1 changed files with 193 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

266
velib.py
View File

@ -1,11 +1,22 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import datetime import datetime
import json import json
import logging
import os import os
import requests import requests
import sqlite3 import sqlite3
import time import time
# Constants
VELIB_STATIONS_INFOS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json'
VELIB_STATIONS_STATUS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json'
# Set up logging
level = logging.WARNING
if os.environ.get('DEBUG', False):
level = logging.DEBUG
logging.basicConfig(level=level)
def db_init(db_name=None): def db_init(db_name=None):
""" """
@ -26,32 +37,53 @@ def db_init(db_name=None):
conn = sqlite3.connect(os.path.join(db_folder, db_name)) conn = sqlite3.connect(os.path.join(db_folder, db_name))
c = conn.cursor() c = conn.cursor()
# Init tables # Init tables
c.execute("CREATE TABLE IF NOT EXISTS stations(" + c.execute("""
"id INTEGER, " + CREATE TABLE IF NOT EXISTS stations(
"name TEXT, " + id INTEGER,
"address TEXT, " + name TEXT,
"latitude REAL, " + address TEXT,
"longitude REAL, " + latitude REAL,
"banking INTEGER, " + longitude REAL,
"bonus INTEGER, " + banking INTEGER,
"bike_stands INTEGER)") bonus INTEGER,
c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" + bike_stands INTEGER
"station_id INTEGER, " + )
"available_bikes INTEGER, " + """)
"available_ebikes INTEGER, " + c.execute("""
"free_stands INTEGER, " + CREATE TABLE IF NOT EXISTS stationsstats(
"status TEXT, " + station_id INTEGER,
"updated INTEGER, " + available_bikes INTEGER,
"FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)") available_ebikes INTEGER,
c.execute("CREATE TABLE IF NOT EXISTS stationsevents(" + free_stands INTEGER,
"station_id INTEGER, " + status TEXT,
"timestamp INTEGER, " + updated INTEGER,
"event TEXT, " + FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
"FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)") )
c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)"); """)
c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)"); c.execute("""
c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)"); CREATE TABLE IF NOT EXISTS stationsevents(
c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)"); station_id INTEGER,
timestamp INTEGER,
event TEXT,
FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
)
""")
c.execute("""
CREATE INDEX IF NOT EXISTS
stationstats_station_id ON stationsstats (station_id)
""")
c.execute("""
CREATE INDEX IF NOT EXISTS
stationsstats_updated ON stationsstats (updated)
""")
c.execute("""
CREATE INDEX IF NOT EXISTS
stationsevents_station_id ON stationsevents (station_id)
""")
c.execute("""
CREATE INDEX IF NOT EXISTS
stationsevents_timestamp ON stationsevents (timestamp)
""")
conn.commit() conn.commit()
return conn return conn
@ -63,16 +95,39 @@ def update_stations(conn):
:param conn: Database connection. :param conn: Database connection.
""" """
c = conn.cursor() c = conn.cursor()
database_stations = {i[0]: i
for i in
c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()}
req_stations = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json') logging.info('Get all stations from database...')
database_stations = {
i[0]: i
for i in c.execute("""
SELECT
id,
name,
address,
latitude,
longitude,
banking,
bonus,
bike_stands
FROM stations
""").fetchall()
}
logging.info('Get updated Velib stations from API...')
req_stations = requests.get(VELIB_STATIONS_INFOS_URL)
stations = { stations = {
station['stationCode']: station station['stationCode']: station
for station in req_stations.json()['data']['stations'] for station in req_stations.json()['data']['stations']
} }
req_status = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json') req_status = requests.get(VELIB_STATIONS_STATUS_URL)
# List of SQL queries to perform for
events = [] # events happening on stations (temporary closure etc)
stations_update = [] # Update of stations (such as new stands number)
new_stations = [] # New stations to add to the list
stats = [] # Current stats of the station
logging.info('Processing fetched stations')
for station in req_status.json()['data']['stations']: for station in req_status.json()['data']['stations']:
uid = station["stationCode"] uid = station["stationCode"]
try: try:
@ -98,40 +153,39 @@ def update_stations(conn):
"new_value": stations[uid]["capacity"]}) "new_value": stations[uid]["capacity"]})
# If diff was found # If diff was found
if len(event) > 0: if len(event) > 0:
# Update stations_update.append(
c.execute("UPDATE " + (
"stations " + stations[uid]['name'],
"SET name=?, latitude=?, longitude=?, " + stations[uid]['lat'],
"banking=?, bike_stands=? WHERE id=?", stations[uid]['lon'],
(stations[uid]['name'], None,
stations[uid]['lat'], stations[uid]['capacity'],
stations[uid]['lon'], uid
None, )
stations[uid]['capacity'], )
uid)) events.append(
# And insert event in the table (
c.execute("INSERT INTO " + uid,
"stationsevents(station_id, timestamp, event) " + int(time.time()),
"VALUES(?, ?, ?)", json.dumps(event)
(uid, )
int(time.time()), )
json.dumps(event)))
except KeyError: except KeyError:
c.execute("INSERT INTO " + # Station is unknown, add it
"stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " + new_stations.append(
"VALUES(?, ?, ?, ?, ?, ?, ?, ?)", (
(uid, uid,
stations[uid]['name'], stations[uid]['name'],
"", # Not available "", # Not available
stations[uid]['lat'], stations[uid]['lat'],
stations[uid]['lon'], stations[uid]['lon'],
None, # Not available None, # Not available
False, # Not available False, # Not available
stations[uid]["capacity"])) stations[uid]["capacity"]
except TypeError: )
conn.rollback() )
return
# Add stats
numEBikesAvailable = ( numEBikesAvailable = (
station['numBikesAvailable'] station['numBikesAvailable']
- next( - next(
@ -140,16 +194,82 @@ def update_stations(conn):
if 'ebike' in x if 'ebike' in x
) )
) )
c.execute("INSERT INTO " + stats.append(
"stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " + (
"VALUES(?, ?, ?, ?, ?, ?)", uid,
(uid, station['numBikesAvailable'],
station['numBikesAvailable'], numEBikesAvailable,
numEBikesAvailable, station['numDocksAvailable'],
station['numDocksAvailable'], None,
None, int(time.time()) # Not available, using current timestamp
int(time.time()))) # Not available, using current timestamp )
conn.commit() )
# Update stations
logging.info('Updating stations in db...')
c.executemany(
"""
UPDATE
stations
SET
name=?, latitude=?, longitude=?,
banking=?, bike_stands=?
WHERE id=?
""",
stations_update
)
# Insert events in the table
logging.info('Insert stations events in db...')
c.executemany(
"""
INSERT INTO
stationsevents(station_id, timestamp, event)
VALUES(?, ?, ?)
""",
events
)
# Add the missing stations to database
logging.info('Insert missing stations in db...')
c.executemany(
"""
INSERT INTO
stations(
id,
name,
address,
latitude,
longitude,
banking,
bonus,
bike_stands
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
""",
new_stations
)
# Insert stats in database
logging.info('Insert stations stats in db...')
c.executemany(
"""
INSERT INTO
stationsstats(
station_id,
available_bikes,
available_ebikes,
free_stands,
status,
updated
)
VALUES(?, ?, ?, ?, ?, ?)
""",
stats
)
# Commit
conn.commit()
def main(): def main():