Optimize SQL handling
This commit is contained in:
parent
79c0dc3b29
commit
f3894c7c9b
266
velib.py
266
velib.py
@ -1,11 +1,22 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import datetime
|
import datetime
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
# Constants
|
||||||
|
VELIB_STATIONS_INFOS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json'
|
||||||
|
VELIB_STATIONS_STATUS_URL = 'https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json'
|
||||||
|
|
||||||
|
# Set up logging
|
||||||
|
level = logging.WARNING
|
||||||
|
if os.environ.get('DEBUG', False):
|
||||||
|
level = logging.DEBUG
|
||||||
|
logging.basicConfig(level=level)
|
||||||
|
|
||||||
|
|
||||||
def db_init(db_name=None):
|
def db_init(db_name=None):
|
||||||
"""
|
"""
|
||||||
@ -26,32 +37,53 @@ def db_init(db_name=None):
|
|||||||
conn = sqlite3.connect(os.path.join(db_folder, db_name))
|
conn = sqlite3.connect(os.path.join(db_folder, db_name))
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
# Init tables
|
# Init tables
|
||||||
c.execute("CREATE TABLE IF NOT EXISTS stations(" +
|
c.execute("""
|
||||||
"id INTEGER, " +
|
CREATE TABLE IF NOT EXISTS stations(
|
||||||
"name TEXT, " +
|
id INTEGER,
|
||||||
"address TEXT, " +
|
name TEXT,
|
||||||
"latitude REAL, " +
|
address TEXT,
|
||||||
"longitude REAL, " +
|
latitude REAL,
|
||||||
"banking INTEGER, " +
|
longitude REAL,
|
||||||
"bonus INTEGER, " +
|
banking INTEGER,
|
||||||
"bike_stands INTEGER)")
|
bonus INTEGER,
|
||||||
c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" +
|
bike_stands INTEGER
|
||||||
"station_id INTEGER, " +
|
)
|
||||||
"available_bikes INTEGER, " +
|
""")
|
||||||
"available_ebikes INTEGER, " +
|
c.execute("""
|
||||||
"free_stands INTEGER, " +
|
CREATE TABLE IF NOT EXISTS stationsstats(
|
||||||
"status TEXT, " +
|
station_id INTEGER,
|
||||||
"updated INTEGER, " +
|
available_bikes INTEGER,
|
||||||
"FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)")
|
available_ebikes INTEGER,
|
||||||
c.execute("CREATE TABLE IF NOT EXISTS stationsevents(" +
|
free_stands INTEGER,
|
||||||
"station_id INTEGER, " +
|
status TEXT,
|
||||||
"timestamp INTEGER, " +
|
updated INTEGER,
|
||||||
"event TEXT, " +
|
FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
|
||||||
"FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)")
|
)
|
||||||
c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)");
|
""")
|
||||||
c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)");
|
c.execute("""
|
||||||
c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)");
|
CREATE TABLE IF NOT EXISTS stationsevents(
|
||||||
c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)");
|
station_id INTEGER,
|
||||||
|
timestamp INTEGER,
|
||||||
|
event TEXT,
|
||||||
|
FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
c.execute("""
|
||||||
|
CREATE INDEX IF NOT EXISTS
|
||||||
|
stationstats_station_id ON stationsstats (station_id)
|
||||||
|
""")
|
||||||
|
c.execute("""
|
||||||
|
CREATE INDEX IF NOT EXISTS
|
||||||
|
stationsstats_updated ON stationsstats (updated)
|
||||||
|
""")
|
||||||
|
c.execute("""
|
||||||
|
CREATE INDEX IF NOT EXISTS
|
||||||
|
stationsevents_station_id ON stationsevents (station_id)
|
||||||
|
""")
|
||||||
|
c.execute("""
|
||||||
|
CREATE INDEX IF NOT EXISTS
|
||||||
|
stationsevents_timestamp ON stationsevents (timestamp)
|
||||||
|
""")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return conn
|
return conn
|
||||||
|
|
||||||
@ -63,16 +95,39 @@ def update_stations(conn):
|
|||||||
:param conn: Database connection.
|
:param conn: Database connection.
|
||||||
"""
|
"""
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
database_stations = {i[0]: i
|
|
||||||
for i in
|
|
||||||
c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()}
|
|
||||||
|
|
||||||
req_stations = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json')
|
logging.info('Get all stations from database...')
|
||||||
|
database_stations = {
|
||||||
|
i[0]: i
|
||||||
|
for i in c.execute("""
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
address,
|
||||||
|
latitude,
|
||||||
|
longitude,
|
||||||
|
banking,
|
||||||
|
bonus,
|
||||||
|
bike_stands
|
||||||
|
FROM stations
|
||||||
|
""").fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
|
logging.info('Get updated Velib stations from API...')
|
||||||
|
req_stations = requests.get(VELIB_STATIONS_INFOS_URL)
|
||||||
stations = {
|
stations = {
|
||||||
station['stationCode']: station
|
station['stationCode']: station
|
||||||
for station in req_stations.json()['data']['stations']
|
for station in req_stations.json()['data']['stations']
|
||||||
}
|
}
|
||||||
req_status = requests.get('https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json')
|
req_status = requests.get(VELIB_STATIONS_STATUS_URL)
|
||||||
|
|
||||||
|
# List of SQL queries to perform for
|
||||||
|
events = [] # events happening on stations (temporary closure etc)
|
||||||
|
stations_update = [] # Update of stations (such as new stands number)
|
||||||
|
new_stations = [] # New stations to add to the list
|
||||||
|
stats = [] # Current stats of the station
|
||||||
|
|
||||||
|
logging.info('Processing fetched stations')
|
||||||
for station in req_status.json()['data']['stations']:
|
for station in req_status.json()['data']['stations']:
|
||||||
uid = station["stationCode"]
|
uid = station["stationCode"]
|
||||||
try:
|
try:
|
||||||
@ -98,40 +153,39 @@ def update_stations(conn):
|
|||||||
"new_value": stations[uid]["capacity"]})
|
"new_value": stations[uid]["capacity"]})
|
||||||
# If diff was found
|
# If diff was found
|
||||||
if len(event) > 0:
|
if len(event) > 0:
|
||||||
# Update
|
stations_update.append(
|
||||||
c.execute("UPDATE " +
|
(
|
||||||
"stations " +
|
stations[uid]['name'],
|
||||||
"SET name=?, latitude=?, longitude=?, " +
|
stations[uid]['lat'],
|
||||||
"banking=?, bike_stands=? WHERE id=?",
|
stations[uid]['lon'],
|
||||||
(stations[uid]['name'],
|
None,
|
||||||
stations[uid]['lat'],
|
stations[uid]['capacity'],
|
||||||
stations[uid]['lon'],
|
uid
|
||||||
None,
|
)
|
||||||
stations[uid]['capacity'],
|
)
|
||||||
uid))
|
events.append(
|
||||||
# And insert event in the table
|
(
|
||||||
c.execute("INSERT INTO " +
|
uid,
|
||||||
"stationsevents(station_id, timestamp, event) " +
|
int(time.time()),
|
||||||
"VALUES(?, ?, ?)",
|
json.dumps(event)
|
||||||
(uid,
|
)
|
||||||
int(time.time()),
|
)
|
||||||
json.dumps(event)))
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
c.execute("INSERT INTO " +
|
# Station is unknown, add it
|
||||||
"stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " +
|
new_stations.append(
|
||||||
"VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
|
(
|
||||||
(uid,
|
uid,
|
||||||
stations[uid]['name'],
|
stations[uid]['name'],
|
||||||
"", # Not available
|
"", # Not available
|
||||||
stations[uid]['lat'],
|
stations[uid]['lat'],
|
||||||
stations[uid]['lon'],
|
stations[uid]['lon'],
|
||||||
None, # Not available
|
None, # Not available
|
||||||
False, # Not available
|
False, # Not available
|
||||||
stations[uid]["capacity"]))
|
stations[uid]["capacity"]
|
||||||
except TypeError:
|
)
|
||||||
conn.rollback()
|
)
|
||||||
return
|
|
||||||
|
|
||||||
|
# Add stats
|
||||||
numEBikesAvailable = (
|
numEBikesAvailable = (
|
||||||
station['numBikesAvailable']
|
station['numBikesAvailable']
|
||||||
- next(
|
- next(
|
||||||
@ -140,16 +194,82 @@ def update_stations(conn):
|
|||||||
if 'ebike' in x
|
if 'ebike' in x
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
c.execute("INSERT INTO " +
|
stats.append(
|
||||||
"stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " +
|
(
|
||||||
"VALUES(?, ?, ?, ?, ?, ?)",
|
uid,
|
||||||
(uid,
|
station['numBikesAvailable'],
|
||||||
station['numBikesAvailable'],
|
numEBikesAvailable,
|
||||||
numEBikesAvailable,
|
station['numDocksAvailable'],
|
||||||
station['numDocksAvailable'],
|
None,
|
||||||
None,
|
int(time.time()) # Not available, using current timestamp
|
||||||
int(time.time()))) # Not available, using current timestamp
|
)
|
||||||
conn.commit()
|
)
|
||||||
|
|
||||||
|
# Update stations
|
||||||
|
logging.info('Updating stations in db...')
|
||||||
|
c.executemany(
|
||||||
|
"""
|
||||||
|
UPDATE
|
||||||
|
stations
|
||||||
|
SET
|
||||||
|
name=?, latitude=?, longitude=?,
|
||||||
|
banking=?, bike_stands=?
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
stations_update
|
||||||
|
)
|
||||||
|
|
||||||
|
# Insert events in the table
|
||||||
|
logging.info('Insert stations events in db...')
|
||||||
|
c.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO
|
||||||
|
stationsevents(station_id, timestamp, event)
|
||||||
|
VALUES(?, ?, ?)
|
||||||
|
""",
|
||||||
|
events
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add the missing stations to database
|
||||||
|
logging.info('Insert missing stations in db...')
|
||||||
|
c.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO
|
||||||
|
stations(
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
address,
|
||||||
|
latitude,
|
||||||
|
longitude,
|
||||||
|
banking,
|
||||||
|
bonus,
|
||||||
|
bike_stands
|
||||||
|
)
|
||||||
|
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
new_stations
|
||||||
|
)
|
||||||
|
|
||||||
|
# Insert stats in database
|
||||||
|
logging.info('Insert stations stats in db...')
|
||||||
|
c.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO
|
||||||
|
stationsstats(
|
||||||
|
station_id,
|
||||||
|
available_bikes,
|
||||||
|
available_ebikes,
|
||||||
|
free_stands,
|
||||||
|
status,
|
||||||
|
updated
|
||||||
|
)
|
||||||
|
VALUES(?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
stats
|
||||||
|
)
|
||||||
|
|
||||||
|
# Commit
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user