phyks/velibdataset
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move to pybikes

Browse Source
This commit is contained in:
Lucas Verney 2018-01-15 13:58:39 +01:00
parent 0e59799f05
commit d040c1ddf5
1 changed files with 53 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
velib.py
View File

@ -3,20 +3,20 @@ from config import *
import datetime import datetime
import json import json
import requests
import os import os
import pybikes
import sqlite3 import sqlite3
import sys
import time import time
def db_init(): def db_init(db_name=None):
""" """
Initialize a database connection, initialize the tables. Initialize a database connection, initialize the tables.
Returns a new connection. Returns a new connection.
""" """
now = datetime.datetime.now() now = datetime.datetime.now()
if db_name is None:
db_name = "week_%s.db" % now.strftime("%V") db_name = "week_%s.db" % now.strftime("%V")
db_folder = os.path.join( db_folder = os.path.join(
'data', 'data',
@ -29,7 +29,7 @@ def db_init():
c = conn.cursor() c = conn.cursor()
# Init tables # Init tables
c.execute("CREATE TABLE IF NOT EXISTS stations(" + c.execute("CREATE TABLE IF NOT EXISTS stations(" +
"id INTEGER PRIMARY KEY, " + "id INTEGER, " +
"name TEXT, " + "name TEXT, " +
"address TEXT, " + "address TEXT, " +
"latitude REAL, " + "latitude REAL, " +
@ -40,6 +40,7 @@ def db_init():
c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" + c.execute("CREATE TABLE IF NOT EXISTS stationsstats(" +
"station_id INTEGER, " + "station_id INTEGER, " +
"available_bikes INTEGER, " + "available_bikes INTEGER, " +
"available_ebikes INTEGER, " +
"free_stands INTEGER, " + "free_stands INTEGER, " +
"status TEXT, " + "status TEXT, " +
"updated INTEGER, " + "updated INTEGER, " +
@ -57,116 +58,90 @@ def db_init():
return conn return conn
def retrieve_stations(): def update_stations(conn):
"""
Retrieve list of stations.
Returns the new stations list.
"""
# Fetch the endpoint
r = requests.get(api_endpoint,
params={"apiKey": api_key, "contract": contract})
# Handle the JSON response
try:
stations_list_json = json.loads(r.text)
except ValueError:
sys.exit("Got invalid JSON payload:\n" + r.text)
stations_list = []
for station in stations_list_json:
stations_list.append(station)
return stations_list
def update_stations():
""" """
Update the stored station list. Update the stored station list.
:param conn: Database connection.
""" """
conn = db_init()
c = conn.cursor() c = conn.cursor()
stations = retrieve_stations()
database_stations = {i[0]: i database_stations = {i[0]: i
for i in for i in
c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()} c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()}
for station in stations:
velib = pybikes.get("velib")
velib.update()
for station in velib.stations:
try: try:
# Get old station entry if it exists # Get old station entry if it exists
old_station = database_stations[station["number"]] old_station = database_stations[station.extra["uid"]]
# Diff the two stations # Diff the two stations
event = [] event = []
if station["name"] != old_station[1]: if station.name != old_station[1]:
event.append({"key": "name", event.append({"key": "name",
"old_value": old_station[1], "old_value": old_station[1],
"new_value": station["name"]}) "new_value": station.name})
if station["address"] != old_station[2]: if station.latitude != old_station[3]:
event.append({"key": "address",
"old_value": old_station[2],
"new_value": station["address"]})
if station["position"]["lat"] != old_station[3]:
event.append({"key": "latitude", event.append({"key": "latitude",
"old_value": old_station[3], "old_value": old_station[3],
"new_value": station["position"]["lat"]}) "new_value": station.latitude})
if station["position"]["lng"] != old_station[4]: if station.longitude != old_station[4]:
event.append({"key": "longitude", event.append({"key": "longitude",
"old_value": old_station[4], "old_value": old_station[4],
"new_value": station["position"]["lng"]}) "new_value": station.longitude})
if station["banking"] != old_station[5]: if station.extra["banking"] != old_station[5]:
event.append({"key": "banking", event.append({"key": "banking",
"old_value": old_station[5], "old_value": old_station[5],
"new_value": station["banking"]}) "new_value": station.extra["banking"]})
if station["bonus"] != old_station[6]: if station.extra["slots"] != old_station[7]:
event.append({"key": "bonus",
"old_value": old_station[6],
"new_value": station["bonus"]})
if station["bike_stands"] != old_station[7]:
event.append({"key": "bike_stands", event.append({"key": "bike_stands",
"old_value": old_station[7], "old_value": old_station[7],
"new_value": station["bike_stands"]}) "new_value": station.extra["slots"]})
# If diff was found # If diff was found
if len(event) > 0: if len(event) > 0:
# Update # Update
c.execute("UPDATE " + c.execute("UPDATE " +
"stations " + "stations " +
"SET name=?, address=?, latitude=?, longitude=?, " + "SET name=?, latitude=?, longitude=?, " +
"banking=?, bonus=?, bike_stands=? WHERE id=?", "banking=?, bike_stands=? WHERE id=?",
(station["name"], (station.name,
station["address"], station.latitude,
station["position"]["lat"], station.longitude,
station["position"]["lng"], station.extra["banking"],
station["banking"], station.extra["slots"],
station["bonus"], station.extra["uid"]))
station["bike_stands"],
station["number"]))
# And insert event in the table # And insert event in the table
c.execute("INSERT INTO " + c.execute("INSERT INTO " +
"stationsevents(station_id, timestamp, event) " + "stationsevents(station_id, timestamp, event) " +
"VALUES(?, ?, ?)", "VALUES(?, ?, ?)",
(station["number"], (station.extra["uid"],
int(time.time()), int(time.time()),
json.dumps(event))) json.dumps(event)))
except KeyError: except KeyError:
c.execute("INSERT INTO " + c.execute("INSERT INTO " +
"stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " + "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?)", "VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
(station["number"], (station.extra["uid"],
station["name"], station.name,
station["address"], "", # Not available
station["position"]["lat"], station.latitude,
station["position"]["lng"], station.longitude,
station["banking"], station.extra["banking"],
station["bonus"], False, # Not available
station["bike_stands"])) station.extra["slots"]))
except TypeError: except TypeError:
conn.rollback() conn.rollback()
return return
c.execute("INSERT INTO " + c.execute("INSERT INTO " +
"stationsstats(station_id, available_bikes, free_stands, status, updated) " + "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " +
"VALUES(?, ?, ?, ?, ?)", "VALUES(?, ?, ?, ?, ?, ?)",
(station["number"], (station.extra["uid"],
station["available_bikes"], station.bikes - station.extra["ebikes"],
station["available_bike_stands"], station.extra["ebikes"],
station["status"], station.free,
station["last_update"])) station.extra["status"],
int(time.time()))) # Not available, using current timestamp
conn.commit() conn.commit()
@ -174,8 +149,9 @@ def main():
""" """
Handle main operations. Handle main operations.
""" """
# Get updated list of stations # Get updated list of stations for smovengo
update_stations() conn = db_init()
update_stations(conn)
if __name__ == "__main__": if __name__ == "__main__":