Track and share issues (work, interruption in routes, parked cars) in realtime on bike lanes! https://cyclo.phyks.me/

works.py 29KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755
  1. #!/usr/bin/env python
  2. """
  3. Import French opendata about works on roads.
  4. """
  5. import datetime
  6. import json
  7. import logging
  8. import os
  9. import sys
  10. SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  11. sys.path.append(os.path.abspath(os.path.join(SCRIPT_DIRECTORY, '..', '..')))
  12. import arrow
  13. import pyproj
  14. import requests
  15. from functools import partial
  16. from shapely.geometry import (LineString, MultiPolygon, MultiLineString,
  17. MultiPoint, Point)
  18. from shapely.geometry import mapping, shape
  19. from shapely.ops import transform
  20. from server.models import db, Report
  21. from server.tools import UTC_now
  22. level = logging.WARN
  23. RAISE_ON_EXCEPT = False
  24. if 'DEBUG' in os.environ:
  25. level = logging.INFO
  26. if 'RAISE_ON_EXCEPT' in os.environ:
  27. RAISE_ON_EXCEPT = True
  28. logging.basicConfig(level=level)
  29. # Same as in src/constants.js
  30. REPORT_DOWNVOTES_THRESHOLD = 1
  31. def preprocess_hauts_de_seine(data):
  32. out = []
  33. for item in data:
  34. try:
  35. fields = item['fields']
  36. new_item = {
  37. 'fields': fields,
  38. 'geometry': {
  39. 'type': 'Point',
  40. # Use lng, lat
  41. 'coordinates': fields['geo_point_2d'][::-1]
  42. },
  43. 'recordid': item['recordid'],
  44. 'source': 'opendata-hauts-de-seine'
  45. }
  46. new_fields = new_item['fields']
  47. # Guess start date / end date
  48. # If the item is in the data, it should be valid for the current
  49. # trimester.
  50. now = datetime.datetime.now()
  51. new_fields['date_debut'] = arrow.get(
  52. datetime.datetime(
  53. now.year,
  54. max(1, min((now.month // 3) * 3, 12)),
  55. 1
  56. )
  57. ).isoformat()
  58. new_fields['date_fin'] = arrow.get(
  59. datetime.datetime(
  60. now.year,
  61. max(1, min((now.month // 3 + 1) * 3, 12)),
  62. 1
  63. )
  64. ).isoformat()
  65. out.append(new_item)
  66. except KeyError as exc:
  67. logging.warning(
  68. 'Invalid item %s in Hauts-de-Seine data: %s.',
  69. item.get('recordid', '?'),
  70. exc
  71. )
  72. continue
  73. return out
  74. def preprocess_lille(data):
  75. out = []
  76. for item in data:
  77. try:
  78. fields = item['fields']
  79. new_item = {
  80. 'fields': fields,
  81. 'geometry': {
  82. 'type': 'Point',
  83. # Use lng, lat
  84. 'coordinates': fields['geo_point_2d'][::-1]
  85. },
  86. 'recordid': item['recordid'],
  87. 'source': 'opendata-lille'
  88. }
  89. new_fields = new_item['fields']
  90. # Homogeneize geo_shape
  91. new_fields['geo_shape'] = new_item['geometry']
  92. # Homogeneize start date spelling
  93. new_fields['date_debut'] = new_fields['date_demarrage']
  94. out.append(new_item)
  95. except KeyError as exc:
  96. logging.warning(
  97. 'Invalid item %s in Lille data: %s.',
  98. item.get('recordid', '?'),
  99. exc
  100. )
  101. continue
  102. return out
  103. def preprocess_loiret(data):
  104. out = []
  105. if not 'features' in data:
  106. logging.warning('Invalid data for Loiret.')
  107. return out
  108. for item in data['features']:
  109. try:
  110. if 'paths' in item['geometry']:
  111. paths = item['geometry']['paths']
  112. else:
  113. paths = [
  114. [
  115. [item['geometry']['x'], item['geometry']['y']]
  116. ]
  117. ]
  118. # In Loiret, multiple paths are for multiple LineStrings
  119. for path in paths:
  120. if len(path) == 1:
  121. geo_shape = {
  122. 'type': 'Point',
  123. 'coordinates': path[0]
  124. }
  125. else:
  126. geo_shape = {
  127. 'type': 'LineString',
  128. 'coordinates': path
  129. }
  130. new_item = {
  131. 'fields': item['attributes'],
  132. 'geometry': shape(geo_shape).centroid,
  133. 'recordid': item['attributes']['OBJECTID'],
  134. 'source': 'opendata-loiret'
  135. }
  136. new_fields = new_item['fields']
  137. # Homogeneize geo_shape
  138. new_fields['geo_shape'] = geo_shape
  139. # Homogeneize start and end date spelling
  140. new_fields['date_debut'] = arrow.get(
  141. float(new_fields['STARTDATE']) / 1000
  142. ).isoformat()
  143. new_fields['date_fin'] = arrow.get(
  144. float(new_fields['ENDDATE']) / 1000
  145. ).isoformat()
  146. out.append(new_item)
  147. except KeyError as exc:
  148. logging.warning(
  149. 'Invalid item %s in Loiret data: %s.',
  150. item.get('attributes', {}).get('OBJECTID', '?'),
  151. exc
  152. )
  153. if RAISE_ON_EXCEPT:
  154. raise
  155. continue
  156. return out
  157. def preprocess_lyon(data):
  158. out = []
  159. if not 'features' in data:
  160. logging.warning('Invalid data for Lyon.')
  161. return out
  162. for item in data['features']:
  163. try:
  164. new_item = {
  165. 'fields': item['properties'],
  166. 'geometry': shape(item['geometry']).centroid,
  167. 'recordid': item['properties']['identifiant'],
  168. 'source': 'opendata-lyon'
  169. }
  170. new_fields = new_item['fields']
  171. # Homogeneize geo_shape
  172. new_fields['geo_shape'] = item['geometry']
  173. # Homogeneize start date and end date spelling
  174. new_fields['date_debut'] = new_fields['debutchantier']
  175. new_fields['date_fin'] = new_fields['finchantier']
  176. out.append(new_item)
  177. except KeyError as exc:
  178. logging.warning(
  179. 'Invalid item %s in Lyon data: %s.',
  180. item.get('properties', {}).get('identifiant', '?'),
  181. exc
  182. )
  183. if RAISE_ON_EXCEPT:
  184. raise
  185. continue
  186. return out
  187. def preprocess_montpellier(data):
  188. out = []
  189. if not 'features' in data:
  190. logging.warning('Invalid data for Montpellier.')
  191. return out
  192. for item in data['features']:
  193. try:
  194. new_item = {
  195. 'fields': item['properties'],
  196. 'geometry': shape(item['geometry']).centroid,
  197. 'recordid': item['properties']['numero'],
  198. 'source': 'opendata-montpellier'
  199. }
  200. new_fields = new_item['fields']
  201. # Homogeneize geo_shape
  202. new_fields['geo_shape'] = item['geometry']
  203. # Homogeneize start date and end date spelling
  204. new_fields['date_debut'] = new_fields['datedebut']
  205. new_fields['date_fin'] = new_fields['datefin']
  206. out.append(new_item)
  207. except KeyError as exc:
  208. logging.warning(
  209. 'Invalid item %s in Montpellier data: %s.',
  210. item.get('properties', {}).get('numero', '?'),
  211. exc
  212. )
  213. if RAISE_ON_EXCEPT:
  214. raise
  215. continue
  216. return out
  217. def preprocess_nancy(data):
  218. out = []
  219. if not 'features' in data:
  220. logging.warning('Invalid data for Nancy.')
  221. return out
  222. for item in data['features']:
  223. try:
  224. geometry = {
  225. 'type': 'Point',
  226. 'coordinates': [
  227. item['geometry']['x'],
  228. item['geometry']['y']
  229. ]
  230. }
  231. new_item = {
  232. 'fields': item['attributes'],
  233. 'geometry': geometry,
  234. 'recordid': item['attributes']['OBJECTID'],
  235. 'source': 'opendata-nancy'
  236. }
  237. new_fields = new_item['fields']
  238. # Homogeneize geo_shape
  239. new_fields['geo_shape'] = geometry
  240. # Homogeneize start and end date spelling
  241. if not new_fields['DATE_DEBUT'] or not new_fields['DATE_FIN']:
  242. # Invalid start / end date
  243. continue
  244. new_fields['date_debut'] = arrow.get(
  245. float(new_fields['DATE_DEBUT']) / 1000
  246. ).isoformat()
  247. new_fields['date_fin'] = arrow.get(
  248. float(new_fields['DATE_FIN']) / 1000
  249. ).isoformat()
  250. out.append(new_item)
  251. except KeyError as exc:
  252. logging.warning(
  253. 'Invalid item %s in Nancy data: %s.',
  254. item.get('attributes', {}).get('OBJECTID', '?'),
  255. exc
  256. )
  257. if RAISE_ON_EXCEPT:
  258. raise
  259. continue
  260. return out
  261. def preprocess_paris(data):
  262. out = []
  263. for item in data:
  264. try:
  265. new_item = {
  266. 'fields': item['fields'],
  267. 'geometry': item['geometry'],
  268. 'recordid': item['recordid'],
  269. 'source': 'opendata-paris',
  270. }
  271. out.append(new_item)
  272. except KeyError as exc:
  273. logging.warning(
  274. 'Invalid item %s in Paris data: %s.',
  275. item.get('recordid', '?'),
  276. exc
  277. )
  278. if RAISE_ON_EXCEPT:
  279. raise
  280. continue
  281. return out
  282. def preprocess_rennes(data):
  283. out = []
  284. if not 'features' in data:
  285. logging.warning('Invalid data for Rennes.')
  286. return out
  287. for item in data['features']:
  288. try:
  289. new_item = {
  290. 'fields': item['properties'],
  291. 'geometry': shape(item['geometry']),
  292. 'recordid': item['properties']['id'],
  293. 'source': 'opendata-rennes'
  294. }
  295. new_fields = new_item['fields']
  296. # Homogeneize geo_shape
  297. new_fields['geo_shape'] = item['geometry']
  298. # Homogeneize start date spelling
  299. new_fields['date_debut'] = new_fields['date_deb']
  300. out.append(new_item)
  301. except KeyError as exc:
  302. logging.warning(
  303. 'Invalid item %s in Rennes data: %s.',
  304. item.get('properties', {}).get('id', '?'),
  305. exc
  306. )
  307. if RAISE_ON_EXCEPT:
  308. raise
  309. continue
  310. return out
  311. def preprocess_seine_saint_denis(data):
  312. out = []
  313. if not 'features' in data:
  314. logging.warning('Invalid data for Seine-Saint-Denis.')
  315. return out
  316. for item in data['features']:
  317. try:
  318. new_item = {
  319. 'fields': item['properties'],
  320. 'geometry': shape(item['geometry']).centroid,
  321. 'recordid': item['properties']['id'],
  322. 'source': 'opendata-seine_saint_denis'
  323. }
  324. # Homogeneize geo_shape
  325. new_item['fields']['geo_shape'] = item['geometry']
  326. out.append(new_item)
  327. except KeyError as exc:
  328. logging.warning(
  329. 'Invalid item %s in Seine-Saint-Denis data: %s.',
  330. item.get('properties', {}).get('id', '?'),
  331. exc
  332. )
  333. if RAISE_ON_EXCEPT:
  334. raise
  335. continue
  336. return out
  337. def preprocess_sicoval(data):
  338. out = []
  339. if 'error' in data:
  340. logging.warning('Invalid data for Sicoval.')
  341. return out
  342. for item in data:
  343. try:
  344. new_item = {
  345. 'fields': item['fields'],
  346. 'geometry': item['geometry'],
  347. 'recordid': item['recordid'],
  348. 'source': 'opendata-sicoval'
  349. }
  350. new_fields = new_item['fields']
  351. # Homogeneize geo_shape
  352. new_fields['geo_shape'] = new_fields['geoshape2']
  353. # Homogeneize start date and end date spelling
  354. new_fields['date_debut'] = new_fields['startdate']
  355. new_fields['date_fin'] = new_fields['enddate']
  356. out.append(new_item)
  357. except KeyError as exc:
  358. logging.warning(
  359. 'Invalid item %s in Sicoval data: %s.',
  360. item.get('recordid', '?'),
  361. exc
  362. )
  363. if RAISE_ON_EXCEPT:
  364. raise
  365. continue
  366. return out
  367. def preprocess_toulouse(data):
  368. out = []
  369. for item in data:
  370. try:
  371. new_item = {
  372. 'fields': item['fields'],
  373. 'geometry': item['geometry'],
  374. 'recordid': item['recordid'],
  375. 'source': 'opendata-toulouse'
  376. }
  377. new_fields = new_item['fields']
  378. # Homogeneize geo_shape
  379. new_fields['geo_shape'] = item['geometry']
  380. # Homogeneize start date and end date spelling
  381. new_fields['date_debut'] = new_fields['datedebut']
  382. new_fields['date_fin'] = new_fields['datefin']
  383. out.append(new_item)
  384. except KeyError as exc:
  385. logging.warning(
  386. 'Invalid item %s in Toulouse data: %s.',
  387. item.get('recordid', '?'),
  388. exc
  389. )
  390. if RAISE_ON_EXCEPT:
  391. raise
  392. continue
  393. return out
  394. def preprocess_versailles(data):
  395. out = []
  396. if not 'features' in data:
  397. logging.warning('Invalid data for Versailles.')
  398. return out
  399. for item in data['features']:
  400. try:
  401. if 'paths' in item['geometry']:
  402. paths = item['geometry']['paths']
  403. else:
  404. paths = [
  405. [
  406. [item['geometry']['x'], item['geometry']['y']]
  407. ]
  408. ]
  409. # In Versailles, multiple paths are for multiple LineStrings
  410. for path in paths:
  411. if len(path) == 1:
  412. geometry = {
  413. 'type': 'Point',
  414. 'coordinates': path[0]
  415. }
  416. else:
  417. geometry = {
  418. 'type': 'LineString',
  419. 'coordinates': path
  420. }
  421. new_item = {
  422. 'fields': item['attributes'],
  423. 'geometry': shape(geometry).centroid,
  424. 'recordid': item['attributes']['OBJECTID'],
  425. 'source': 'opendata-versailles'
  426. }
  427. new_fields = new_item['fields']
  428. # Homogeneize geo_shape
  429. new_fields['geo_shape'] = geometry
  430. # Homogeneize start and end date spelling
  431. new_fields['date_debut'] = arrow.get(
  432. float(new_fields['STARTDATE']) / 1000
  433. ).isoformat()
  434. new_fields['date_fin'] = arrow.get(
  435. float(new_fields['ENDDATE']) / 1000
  436. ).isoformat()
  437. out.append(new_item)
  438. except KeyError as exc:
  439. logging.warning(
  440. 'Invalid item %s in Versailles data: %s.',
  441. item.get('attributes', {}).get('OBJECTID', '?'),
  442. exc
  443. )
  444. if RAISE_ON_EXCEPT:
  445. raise
  446. continue
  447. return out
  448. MIN_DISTANCE_REPORT_DETAILS = 40 # Distance in meters, same as in constants.js
  449. OPENDATA_URLS = {
  450. # Work in Hauts de Seine
  451. # https://opendata.hauts-de-seine.fr/explore/dataset/travaux-sur-la-voirie-departementale-lignes/information/
  452. # Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
  453. "hauts-de-seine": {
  454. "preprocess": preprocess_hauts_de_seine,
  455. "url": "https://opendata.hauts-de-seine.fr/explore/dataset/travaux-sur-la-voirie-departementale-lignes/download/?format=json&timezone=Europe/Berlin",
  456. },
  457. # Work in Lille
  458. # https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/
  459. # Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
  460. "lille": {
  461. "preprocess": preprocess_lille,
  462. "url": "https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/download/?format=json&timezone=Europe/Berlin"
  463. },
  464. # Work in Loiret
  465. # https://open-loiret.opendata.arcgis.com/datasets/74c95548589d4ddeb3fcf094f7d61a67_1?geometry=0.609%2C47.694%2C3.245%2C48.016&orderBy=BLOCKNM
  466. # Custom license
  467. "loiret": {
  468. "preprocess": preprocess_loiret,
  469. "url": "https://services2.arcgis.com/IEzPuQhCEVCtkVvT/arcgis/rest/services/Travaux_routiers/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson",
  470. },
  471. # Work in Lyon
  472. # https://data.grandlyon.com/equipements/chantiers-perturbants-de-la-mftropole-de-lyon/
  473. # Licence Ouverte : https://download.data.grandlyon.com/files/grandlyon/LicenceOuverte.pdf
  474. "lyon": {
  475. "preprocess": preprocess_lyon,
  476. "url": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0&outputformat=GEOJSON&maxfeatures=30&request=GetFeature&typename=pvo_patrimoine_voirie.pvochantierperturbant&SRSNAME=urn:ogc:def:crs:EPSG::4171",
  477. },
  478. # Work in Montpellier
  479. # http://data.montpellier3m.fr/dataset/chantiers-genants-de-montpellier
  480. # Licence ODbL : http://opendefinition.org/licenses/odc-odbl/
  481. "montpellier": {
  482. "preprocess": preprocess_montpellier,
  483. "url": "http://data.montpellier3m.fr/sites/default/files/ressources/MMM_MMM_Chantiers.json"
  484. },
  485. # Work in Nancy
  486. # http://opendata.grandnancy.eu/jeux-de-donnees/detail-dune-fiche-de-donnees/?tx_icsoddatastore_pi1%5Bkeywords%5D=travaux&tx_icsoddatastore_pi1%5Buid%5D=63&tx_icsoddatastore_pi1%5BreturnID%5D=447
  487. # Licence libre / Licence Ouverte (Etalab)
  488. "nancy": {
  489. "preprocess": preprocess_nancy,
  490. "url": "https://geoservices.grand-nancy.org/arcgis/rest/services/public/VOIRIE_Info_Travaux_Niveau/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
  491. },
  492. # Work in Paris
  493. # https://opendata.paris.fr/explore/dataset/chantiers-perturbants/
  494. # Licence ODbL : http://opendatacommons.org/licenses/odbl/
  495. "paris": {
  496. "preprocess": preprocess_paris,
  497. "url": "https://opendata.paris.fr/explore/dataset/chantiers-perturbants/download/?format=json&timezone=Europe/Berlin",
  498. },
  499. # Work in Rennes
  500. # http://travaux.data.rennesmetropole.fr/
  501. # Usage libre / Licence ODbL : http://opendatacommons.org/licenses/odbl/
  502. "rennes": {
  503. "preprocess": preprocess_rennes,
  504. "url": "http://travaux.data.rennesmetropole.fr/api/roadworks?epsg=4326"
  505. },
  506. # Work in Seine-Saint-Denis
  507. # https://geo.data.gouv.fr/fr/datasets/12504debb9bb73e717ad710a746541ebf817d98c
  508. # Licence Ouverte : https://www.etalab.gouv.fr/licence-ouverte-open-licence
  509. "seine-saint-denis": {
  510. "preprocess": preprocess_seine_saint_denis,
  511. "url": "https://geoportail93.fr/SERV/DATA/?TYPENAME=1570&FORMAT=GEOJSON&COL=id;titre&FORMAT=GEOJSON&COL=ALL&MODE=2"
  512. },
  513. # Work in Sicoval (South of Toulouse)
  514. # https://data.sicoval.fr/explore/dataset/travauxincidents/
  515. # Licence Ouverte v2.0 (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2017/04/ETALAB-Licence-Ouverte-v2.0.pdf
  516. "sicoval": {
  517. "preprocess": preprocess_sicoval,
  518. "url": "https://data.sicoval.fr/explore/dataset/travauxincidents/download/?format=json&timezone=Europe/Berlin"
  519. },
  520. # Work in Toulouse
  521. # https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/
  522. # Licence ODbL : http://opendatacommons.org/licenses/odbl/
  523. "toulouse": {
  524. "preprocess": preprocess_toulouse,
  525. "url": "https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/download/?format=json&timezone=Europe/Berlin",
  526. },
  527. # Work in Versailles
  528. # Licence Ouverte (Etalab)
  529. "versailles-blocks": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_0
  530. "preprocess": preprocess_versailles,
  531. "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
  532. },
  533. "versailles-closures": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_1
  534. "preprocess": preprocess_versailles,
  535. "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
  536. },
  537. "versailles-detours": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_2
  538. "preprocess": preprocess_versailles,
  539. "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
  540. },
  541. }
  542. REPORT_TYPE = 'interrupt'
  543. # Projection from WGS84 (GPS) to Lambert93 (for easy buffering of polygons)
  544. project = partial(
  545. pyproj.transform,
  546. pyproj.Proj(init='epsg:4326'), # source coordinates system
  547. pyproj.Proj(init='epsg:2154') # destination coordinates system
  548. )
  549. def process_opendata(name, data, report_type=REPORT_TYPE):
  550. # Coordinates of current reports in db, as shapely Points in Lambert93
  551. current_reports_points = []
  552. active_reports_from_db = Report.select().where(
  553. # Load reports from db of the current type
  554. (Report.type == report_type) &
  555. (
  556. # Either with an expiration_datetime in the future
  557. (
  558. (Report.expiration_datetime is not None) &
  559. (Report.expiration_datetime > UTC_now())
  560. ) |
  561. # Or without expiration_datetime but which are still active (shown
  562. # on the map)
  563. (
  564. (Report.expiration_datetime is None) &
  565. (Report.downvotes < REPORT_DOWNVOTES_THRESHOLD)
  566. )
  567. )
  568. )
  569. for report in active_reports_from_db:
  570. current_reports_points.append(
  571. transform(project, Point(report.lng, report.lat))
  572. )
  573. for item in data:
  574. try:
  575. fields = item['fields']
  576. # Check that the work is currently being done
  577. now = arrow.now('Europe/Paris')
  578. if fields['date_debut']:
  579. start_date = arrow.get(fields['date_debut'])
  580. else:
  581. # Defaults to now if start date is unknown
  582. start_date = arrow.get(now)
  583. if fields['date_fin']:
  584. end_date = arrow.get(fields['date_fin'])
  585. else:
  586. # Defaults to in a week if start date is unknown
  587. end_date = arrow.get(now).shift(days=+7)
  588. if not (start_date < now < end_date):
  589. logging.info(
  590. 'Ignoring record %s, work not currently in progress.',
  591. item['recordid']
  592. )
  593. continue
  594. # Report geographical shape
  595. if 'geo_shape' in fields:
  596. maybe_multi_geo_shape = shape(fields['geo_shape'])
  597. else:
  598. maybe_multi_geo_shape = shape(item['geometry'])
  599. geo_shapes = []
  600. if (
  601. isinstance(maybe_multi_geo_shape, MultiPolygon)
  602. or isinstance(maybe_multi_geo_shape, MultiPoint)
  603. ):
  604. # Split MultiPolygon into multiple Polygon
  605. # Same for MultiPoint
  606. positions = [
  607. p.centroid
  608. for p in maybe_multi_geo_shape
  609. ]
  610. geo_shapes = [
  611. p
  612. for p in maybe_multi_geo_shape
  613. ]
  614. elif isinstance(maybe_multi_geo_shape, MultiLineString):
  615. # Split MultiLineString into multiple LineString
  616. positions = [
  617. p.interpolate(0.5, normalized=True)
  618. for p in maybe_multi_geo_shape
  619. ]
  620. geo_shapes = [
  621. p
  622. for p in maybe_multi_geo_shape
  623. ]
  624. elif isinstance(maybe_multi_geo_shape, LineString):
  625. # LineString, interpolate midpoint
  626. positions = [
  627. maybe_multi_geo_shape.interpolate(0.5, normalized=True)
  628. ]
  629. geo_shapes = [maybe_multi_geo_shape]
  630. else:
  631. # Polygon or Point
  632. positions = [
  633. maybe_multi_geo_shape.centroid
  634. ]
  635. geo_shapes = [maybe_multi_geo_shape]
  636. for (geo_shape, position) in zip(geo_shapes, positions):
  637. # Check if this precise position is already in the database
  638. if transform(project, position) in current_reports_points:
  639. logging.info(
  640. ('Ignoring record %s, a similar report is already in '
  641. 'the database.'),
  642. item['recordid']
  643. )
  644. continue
  645. # Check no similar reports is within the area of the report, up
  646. # to the report distance.
  647. overlap_area = transform(project, geo_shape).buffer(
  648. MIN_DISTANCE_REPORT_DETAILS
  649. )
  650. is_already_inserted = False
  651. for report_point in current_reports_points:
  652. if report_point.within(overlap_area):
  653. # A similar report is already there
  654. is_already_inserted = True
  655. logging.info(
  656. ('Ignoring record %s, a similar report is already '
  657. 'in the database.'),
  658. item['recordid']
  659. )
  660. break
  661. if is_already_inserted:
  662. # Skip this report if a similar one is nearby
  663. continue
  664. # Get the position of the center of the item
  665. lng, lat = position.x, position.y
  666. # Compute expiration datetime
  667. expiration_datetime = end_date.replace(microsecond=0).naive
  668. # Add the report to the db
  669. logging.info('Adding record %s to the database.',
  670. item['recordid'])
  671. Report.create(
  672. type=report_type,
  673. expiration_datetime=expiration_datetime,
  674. lat=lat,
  675. lng=lng,
  676. source=item['source'],
  677. shape_geojson=json.dumps(mapping(geo_shape))
  678. )
  679. except KeyError as exc:
  680. logging.warning(
  681. 'Invalid record %s in %s, missing key: %s',
  682. item.get('recordid', '?'),
  683. name,
  684. exc
  685. )
  686. if __name__ == '__main__':
  687. db.connect()
  688. for name, item in OPENDATA_URLS.items():
  689. logging.info('Processing opendata from %s', name)
  690. try:
  691. r = requests.get(item['url'])
  692. data = r.json()
  693. except (requests.RequestException, ValueError) as exc:
  694. logging.warning('Error while fetching data for %s: %s.',
  695. name, exc)
  696. continue
  697. if item['preprocess']:
  698. data = item['preprocess'](data)
  699. process_opendata(name, data)