Browse Source

Rework SQLite database handling to make the script more efficient

parent
commit
aaa9b19cd3
1 changed files with 106 additions and 55 deletions
  1. 106
    55
      velib.py

+ 106
- 55
velib.py View File

@@ -1,11 +1,17 @@
1 1
 #!/usr/bin/env python3
2 2
 import datetime
3 3
 import json
4
+import logging
4 5
 import os
5 6
 import pybikes
6 7
 import sqlite3
7 8
 import time
8 9
 
10
+level = logging.WARNING
11
+if os.environ.get('DEBUG', False):
12
+    level = logging.DEBUG
13
+logging.basicConfig(level=level)
14
+
9 15
 
10 16
 def db_init(db_name=None):
11 17
     """
@@ -13,6 +19,7 @@ def db_init(db_name=None):
13 19
 
14 20
     Returns a new connection.
15 21
     """
22
+    logging.info('Initialize DB...')
16 23
     now = datetime.datetime.now()
17 24
     if db_name is None:
18 25
         db_name = "week_%s.db" % now.strftime("%V")
@@ -48,10 +55,10 @@ def db_init(db_name=None):
48 55
               "timestamp INTEGER, " +
49 56
               "event TEXT, " +
50 57
               "FOREIGN KEY(station_id) REFERENCES stations(id) ON DELETE CASCADE)")
51
-    c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)");
52
-    c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)");
53
-    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)");
54
-    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)");
58
+    c.execute("CREATE INDEX IF NOT EXISTS stationstats_station_id ON stationsstats (station_id)")
59
+    c.execute("CREATE INDEX IF NOT EXISTS stationsstats_updated ON stationsstats (updated)")
60
+    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_station_id ON stationsevents (station_id)")
61
+    c.execute("CREATE INDEX IF NOT EXISTS stationsevents_timestamp ON stationsevents (timestamp)")
55 62
     conn.commit()
56 63
     return conn
57 64
 
@@ -62,17 +69,35 @@ def update_stations(conn):
62 69
 
63 70
     :param conn: Database connection.
64 71
     """
72
+    logging.info('Get all stations from database...')
65 73
     c = conn.cursor()
66
-    database_stations = {i[0]: i
67
-                         for i in
68
-                         c.execute("SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations").fetchall()}
74
+    database_stations = {
75
+        i[0]: i
76
+        for i in c.execute(
77
+            "SELECT id, name, address, latitude, longitude, banking, bonus, bike_stands FROM stations"
78
+        ).fetchall()
79
+    }
69 80
 
81
+    logging.info('Get updated Velib stations from the API...')
70 82
     velib = pybikes.get("velib")
71 83
     velib.update()
72
-    for station in velib.stations:
84
+    fetched_stations = {
85
+        station.extra['uid']: station
86
+        for station in velib.stations
87
+        if 'uid' in station.extra
88
+    }
89
+
90
+    # List of SQL queries to perform for
91
+    events = []  # events happening on stations (temporary closure etc)
92
+    stations_update = []  # Update of stations (such as new stands number)
93
+    new_stations = []  # New stations to add to the list
94
+    stats = []  # Current stats of the station
95
+
96
+    logging.info('Processing fetched stations...')
97
+    for uid, station in fetched_stations.items():
73 98
         try:
74
-            # Get old station entry if it exists
75
-            old_station = database_stations[station.extra["uid"]]
99
+            old_station = database_stations[uid]
100
+
76 101
             # Diff the two stations
77 102
             event = []
78 103
             if station.name != old_station[1]:
@@ -87,60 +112,86 @@ def update_stations(conn):
87 112
                 event.append({"key": "longitude",
88 113
                               "old_value": old_station[4],
89 114
                               "new_value": station.longitude})
90
-            if station.extra["banking"] != old_station[5]:
115
+            if (
116
+                "banking" in station.extra
117
+                and station.extra["banking"] != old_station[5]
118
+            ):
91 119
                 event.append({"key": "banking",
92 120
                               "old_value": old_station[5],
93 121
                               "new_value": station.extra["banking"]})
94
-            if station.extra["slots"] != old_station[7]:
122
+            if (
123
+                "slots" in station.extra
124
+                and station.extra["slots"] != old_station[7]
125
+            ):
95 126
                 event.append({"key": "bike_stands",
96 127
                               "old_value": old_station[7],
97 128
                               "new_value": station.extra["slots"]})
98 129
             # If diff was found
99 130
             if len(event) > 0:
100
-                # Update
101
-                c.execute("UPDATE " +
102
-                          "stations " +
103
-                          "SET name=?, latitude=?, longitude=?, " +
104
-                          "banking=?, bike_stands=? WHERE id=?",
105
-                          (station.name,
106
-                           station.latitude,
107
-                           station.longitude,
108
-                           station.extra["banking"],
109
-                           station.extra["slots"],
110
-                           station.extra["uid"]))
111
-                # And insert event in the table
112
-                c.execute("INSERT INTO " +
113
-                          "stationsevents(station_id, timestamp, event) " +
114
-                          "VALUES(?, ?, ?)",
115
-                          (station.extra["uid"],
116
-                           int(time.time()),
117
-                           json.dumps(event)))
131
+                stations_update.append(
132
+                    (station.name,
133
+                     station.latitude,
134
+                     station.longitude,
135
+                     station.extra["banking"],
136
+                     station.extra["slots"],
137
+                     uid)
138
+                )
139
+                events.append(
140
+                    (uid, int(time.time()), json.dumps(event))
141
+                )
118 142
         except KeyError:
119
-            c.execute("INSERT INTO " +
120
-                      "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " +
121
-                      "VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
122
-                      (station.extra["uid"],
123
-                       station.name,
124
-                       "",  # Not available
125
-                       station.latitude,
126
-                       station.longitude,
127
-                       station.extra["banking"],
128
-                       False,  # Not available
129
-                       station.extra["slots"]))
130
-        except TypeError:
131
-            conn.rollback()
132
-            return
133
-
134
-        c.execute("INSERT INTO " +
135
-                  "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " +
136
-                  "VALUES(?, ?, ?, ?, ?, ?)",
137
-                  (station.extra["uid"],
138
-                   station.bikes - station.extra["ebikes"],
139
-                   station.extra["ebikes"],
140
-                   station.free,
141
-                   station.extra["status"],
142
-                   int(time.time())))  # Not available, using current timestamp
143
-        conn.commit()
143
+            # Add the station to the database
144
+            new_stations.append(
145
+                (uid,
146
+                 station.name,
147
+                 "",  # Not available
148
+                 station.latitude,
149
+                 station.longitude,
150
+                 station.extra["banking"],
151
+                 False,  # Not available
152
+                 station.extra["slots"])
153
+            )
154
+        stats.append(
155
+            (uid,
156
+             station.bikes - station.extra["ebikes"],
157
+             station.extra["ebikes"],
158
+             station.free,
159
+             station.extra["status"],
160
+             int(time.time()))  # Not available, using current timestamp
161
+        )
162
+
163
+    # Update stations
164
+    logging.info('Updating stations in db...')
165
+    c.executemany(
166
+        "UPDATE stations SET name=?, latitude=?, longitude=?, " +
167
+        "banking=?, bike_stands=? WHERE id=?",
168
+        stations_update
169
+    )
170
+    # Insert events in the table
171
+    logging.info('Storing stations events in db...')
172
+    c.executemany(
173
+        "INSERT INTO stationsevents(station_id, timestamp, event) " +
174
+        "VALUES(?, ?, ?)",
175
+        events
176
+    )
177
+    # Add the station to the database
178
+    logging.info('Storing new stations in db...')
179
+    c.executemany(
180
+        "INSERT INTO " +
181
+        "stations(id, name, address, latitude, longitude, banking, bonus, bike_stands) " +
182
+        "VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
183
+        new_stations
184
+    )
185
+
186
+    # Insert the current state in the db
187
+    logging.info('Storing current stations stats in db...')
188
+    c.executemany(
189
+        "INSERT INTO " +
190
+        "stationsstats(station_id, available_bikes, available_ebikes, free_stands, status, updated) " +
191
+        "VALUES(?, ?, ?, ?, ?, ?)",
192
+        stats
193
+    )
194
+    conn.commit()
144 195
 
145 196
 
146 197
 def main():