Improve OpenData handling for roadwork, clear mention in About section and add OpenData for Loiret
This commit is contained in:
parent
bbca4d88ea
commit
1e7b2b363a
16
README.md
16
README.md
@ -70,14 +70,16 @@ Icons are made from the original works:
|
||||
Beep sound is provided under CC0 license and coming from
|
||||
[freesound.org](https://freesound.org/people/thisusernameis/sounds/426888/).
|
||||
|
||||
The map background is using tiles from <a
|
||||
href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or
|
||||
<a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a
|
||||
href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap
|
||||
contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>.
|
||||
The map background is using tiles from
|
||||
[Carto.com](https://carto.com/location-data-services/basemaps/) or
|
||||
[Thunderforest](http://thunderforest.com/), thanks to [OpenStreetMap
|
||||
contributors](https://www.openstreetmap.org/copyright) and
|
||||
[Leaflet](http://leafletjs.com/).
|
||||
|
||||
Manual location picking uses the awesome API from <a
|
||||
href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>.
|
||||
Manual location picking uses the awesome API from [adresse.data.gouv.fr](https://adresse.data.gouv.fr).
|
||||
|
||||
Details about the sources and licenses of the opendata imported is available
|
||||
in the `scripts/opendata/` scripts.
|
||||
|
||||
|
||||
## Thanks
|
||||
|
||||
@ -21,8 +21,11 @@ from shapely.ops import transform
|
||||
from server.models import db, Report
|
||||
|
||||
level = logging.WARN
|
||||
RAISE_ON_EXCEPT = False
|
||||
if 'DEBUG' in os.environ:
|
||||
level = logging.info
|
||||
level = logging.INFO
|
||||
if 'RAISE_ON_EXCEPT' in os.environ:
|
||||
RAISE_ON_EXCEPT = True
|
||||
logging.basicConfig(level=level)
|
||||
|
||||
|
||||
@ -40,6 +43,53 @@ def preprocess_lille(data):
|
||||
return data
|
||||
|
||||
|
||||
def preprocess_loiret(data):
|
||||
out = []
|
||||
for item in data['features']:
|
||||
try:
|
||||
if 'paths' in item['geometry']:
|
||||
paths = item['geometry']['paths']
|
||||
else:
|
||||
paths = [
|
||||
[
|
||||
[item['geometry']['x'], item['geometry']['y']]
|
||||
]
|
||||
]
|
||||
for path in paths:
|
||||
if len(path) == 1:
|
||||
geometry = {
|
||||
'type': 'Point',
|
||||
'coordinates': path[0]
|
||||
}
|
||||
else:
|
||||
geometry = {
|
||||
'type': 'LineString',
|
||||
'coordinates': path
|
||||
}
|
||||
new_item = {
|
||||
'fields': item['attributes'],
|
||||
'geometry': geometry,
|
||||
'recordid': item['attributes']['OBJECTID']
|
||||
}
|
||||
fields = new_item['fields']
|
||||
# Homogeneize start and end date spelling
|
||||
fields['date_debut'] = arrow.get(
|
||||
float(fields['STARTDATE']) / 1000
|
||||
).isoformat()
|
||||
fields['date_fin'] = arrow.get(
|
||||
float(fields['ENDDATE']) / 1000
|
||||
).isoformat()
|
||||
# Homogeneize geo shape
|
||||
fields['geo_shape'] = geometry
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Loiret data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def preprocess_lyon(data):
|
||||
out = []
|
||||
for item in data['features']:
|
||||
@ -58,6 +108,8 @@ def preprocess_lyon(data):
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Lyon data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -80,6 +132,8 @@ def preprocess_montpellier(data):
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Montpellier data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -113,6 +167,8 @@ def preprocess_nancy(data):
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Nancy data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -134,6 +190,8 @@ def preprocess_rennes(data):
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Rennes data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -153,6 +211,8 @@ def preprocess_seine_saint_denis(data):
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Seine-Saint-Denis data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -168,6 +228,8 @@ def preprocess_sicoval(data):
|
||||
fields['geo_shape'] = fields['geoshape2']
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Sicoval data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return data
|
||||
|
||||
@ -181,6 +243,8 @@ def preprocess_toulouse(data):
|
||||
fields['date_fin'] = fields['datefin']
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Toulouse data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return data
|
||||
|
||||
@ -189,20 +253,45 @@ def preprocess_versailles(data):
|
||||
out = []
|
||||
for item in data['features']:
|
||||
try:
|
||||
new_item = {
|
||||
'fields': item['properties'],
|
||||
'geometry': item['geometry'],
|
||||
'recordid': item['properties']['OBJECTID']
|
||||
}
|
||||
fields = new_item['fields']
|
||||
# Homogeneize start date and end date spelling
|
||||
fields['date_debut'] = fields['STARTDATE']
|
||||
fields['date_fin'] = fields['ENDDATE']
|
||||
# Homogeneize geo shape
|
||||
fields['geo_shape'] = item['geometry']
|
||||
out.append(new_item)
|
||||
if 'paths' in item['geometry']:
|
||||
paths = item['geometry']['paths']
|
||||
else:
|
||||
paths = [
|
||||
[
|
||||
[item['geometry']['x'], item['geometry']['y']]
|
||||
]
|
||||
]
|
||||
for path in paths:
|
||||
if len(path) == 1:
|
||||
geometry = {
|
||||
'type': 'Point',
|
||||
'coordinates': path[0]
|
||||
}
|
||||
else:
|
||||
geometry = {
|
||||
'type': 'LineString',
|
||||
'coordinates': path
|
||||
}
|
||||
new_item = {
|
||||
'fields': item['attributes'],
|
||||
'geometry': geometry,
|
||||
'recordid': item['attributes']['OBJECTID']
|
||||
}
|
||||
fields = new_item['fields']
|
||||
# Homogeneize start and end date spelling
|
||||
fields['date_debut'] = arrow.get(
|
||||
float(fields['STARTDATE']) / 1000
|
||||
).isoformat()
|
||||
fields['date_fin'] = arrow.get(
|
||||
float(fields['ENDDATE']) / 1000
|
||||
).isoformat()
|
||||
# Homogeneize geo shape
|
||||
fields['geo_shape'] = geometry
|
||||
out.append(new_item)
|
||||
except KeyError as exc:
|
||||
logging.warning('Invalid item in Versailles data: %s.', exc)
|
||||
if RAISE_ON_EXCEPT:
|
||||
raise
|
||||
continue
|
||||
return out
|
||||
|
||||
@ -210,54 +299,88 @@ def preprocess_versailles(data):
|
||||
MIN_DISTANCE_REPORT_DETAILS = 40 # Distance in meters, same as in constants.js
|
||||
OPENDATA_URLS = {
|
||||
# Work in Lille
|
||||
# https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/
|
||||
# Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
|
||||
"lille": {
|
||||
"preprocess": preprocess_lille,
|
||||
"url": "https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/download/?format=json&timezone=Europe/Berlin"
|
||||
},
|
||||
# Work in Loiret
|
||||
# https://open-loiret.opendata.arcgis.com/datasets/74c95548589d4ddeb3fcf094f7d61a67_1?geometry=0.609%2C47.694%2C3.245%2C48.016&orderBy=BLOCKNM
|
||||
# Custom license
|
||||
"loiret": {
|
||||
"preprocess": preprocess_loiret,
|
||||
"url": "https://services2.arcgis.com/IEzPuQhCEVCtkVvT/arcgis/rest/services/Travaux_routiers/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson",
|
||||
},
|
||||
# Work in Lyon
|
||||
# https://data.grandlyon.com/equipements/chantiers-perturbants-de-la-mftropole-de-lyon/
|
||||
# Licence Ouverte : https://download.data.grandlyon.com/files/grandlyon/LicenceOuverte.pdf
|
||||
"lyon": {
|
||||
"preprocess": preprocess_lyon,
|
||||
"url": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0&outputformat=GEOJSON&maxfeatures=30&request=GetFeature&typename=pvo_patrimoine_voirie.pvochantierperturbant&SRSNAME=urn:ogc:def:crs:EPSG::4171",
|
||||
},
|
||||
# Work in Montpellier
|
||||
# http://data.montpellier3m.fr/dataset/chantiers-genants-de-montpellier
|
||||
# Licence ODbL : http://opendefinition.org/licenses/odc-odbl/
|
||||
"montpellier": {
|
||||
"preprocess": preprocess_montpellier,
|
||||
"url": "http://data.montpellier3m.fr/sites/default/files/ressources/MMM_MMM_Chantiers.json"
|
||||
},
|
||||
# Work in Nancy
|
||||
# http://opendata.grandnancy.eu/jeux-de-donnees/detail-dune-fiche-de-donnees/?tx_icsoddatastore_pi1%5Bkeywords%5D=travaux&tx_icsoddatastore_pi1%5Buid%5D=63&tx_icsoddatastore_pi1%5BreturnID%5D=447
|
||||
# Licence libre / Licence Ouverte (Etalab)
|
||||
"nancy": {
|
||||
"preprocess": preprocess_nancy,
|
||||
"url": "https://geoservices.grand-nancy.org/arcgis/rest/services/public/VOIRIE_Info_Travaux_Niveau/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
|
||||
},
|
||||
# Work in Paris
|
||||
# https://opendata.paris.fr/explore/dataset/chantiers-perturbants/
|
||||
# Licence ODbL : http://opendatacommons.org/licenses/odbl/
|
||||
"paris": {
|
||||
"preprocess": None,
|
||||
"url": "https://opendata.paris.fr/explore/dataset/chantiers-perturbants/download/?format=json&timezone=Europe/Berlin",
|
||||
},
|
||||
# Work in Rennes
|
||||
# http://travaux.data.rennesmetropole.fr/
|
||||
# Usage libre / Licence ODbL : http://opendatacommons.org/licenses/odbl/
|
||||
"rennes": {
|
||||
"preprocess": preprocess_rennes,
|
||||
"url": "http://travaux.data.rennesmetropole.fr/api/roadworks?epsg=4326"
|
||||
},
|
||||
# Work in Seine-Saint-Denis
|
||||
# https://geo.data.gouv.fr/fr/datasets/12504debb9bb73e717ad710a746541ebf817d98c
|
||||
# Licence Ouverte : https://www.etalab.gouv.fr/licence-ouverte-open-licence
|
||||
"seine-saint-denis": {
|
||||
"preprocess": preprocess_seine_saint_denis,
|
||||
"url": "https://geo.data.gouv.fr/api/geogw/services/5779810963f06a3a8e81541b/feature-types/C1296/download?format=GeoJSON&projection=WGS84"
|
||||
},
|
||||
# Work in Sicoval (South of Toulouse)
|
||||
# https://data.opendatasoft.com/explore/dataset/travauxincidents@sicoval-haute-garonne/
|
||||
# Licence Ouverte v2.0 (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2017/04/ETALAB-Licence-Ouverte-v2.0.pdf
|
||||
"sicoval": {
|
||||
"preprocess": preprocess_sicoval,
|
||||
"url": "https://data.opendatasoft.com/explore/dataset/travauxincidents@sicoval-haute-garonne/download/?format=json&timezone=Europe/Berlin"
|
||||
},
|
||||
# Work in Toulouse
|
||||
# https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/
|
||||
# Licence ODbL : http://opendatacommons.org/licenses/odbl/
|
||||
"toulouse": {
|
||||
"preprocess": preprocess_toulouse,
|
||||
"url": "https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/download/?format=json&timezone=Europe/Berlin",
|
||||
},
|
||||
# Work in Versailles
|
||||
"versailles": {
|
||||
# Licence Ouverte (Etalab)
|
||||
"versailles-blocks": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_0
|
||||
"preprocess": preprocess_versailles,
|
||||
"url": "https://www.data.gouv.fr/en/datasets/r/16f90f14-7896-4463-bc35-9809b55c2c1b"
|
||||
"url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
|
||||
},
|
||||
"versailles-closures": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_1
|
||||
"preprocess": preprocess_versailles,
|
||||
"url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
|
||||
},
|
||||
"versailles-detours": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_2
|
||||
"preprocess": preprocess_versailles,
|
||||
"url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
|
||||
},
|
||||
}
|
||||
REPORT_TYPE = 'interrupt'
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"availableReportsTitle": "The available reports so far are:",
|
||||
"lastReportAdded": "Last report added {fromNow}.",
|
||||
"license": "License",
|
||||
"licenseDescription": "It is released under an <a href=\"https://opensource.org/licenses/MIT\">MIT license</a> (<a href=\"https://framagit.org/phyks/cyclassist\">source code</a>). Icons are based on creations from Wikimedia, Vecteezy, Pixabay or Flaticon. Sounds are based on CC0 works from <a href=\"https://freesound.org/\">freesound.org</a>. The map background is using tiles from <a href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or <a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>. Collected reports are available under <a href=\"https://opendatacommons.org/licenses/odbl/\">ODbL license</a>. Manual location picking uses the awesome API from <a href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>.",
|
||||
"licenseDescription": "It is released under an <a href=\"https://opensource.org/licenses/MIT\">MIT license</a> (<a href=\"https://framagit.org/phyks/cyclassist\">source code</a>). Icons are based on creations from Wikimedia, Vecteezy, Pixabay or Flaticon. Sounds are based on CC0 works from <a href=\"https://freesound.org/\">freesound.org</a>. The map background is using tiles from <a href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or <a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>. Collected reports are available under <a href=\"https://opendatacommons.org/licenses/odbl/\">ODbL license</a>. Manual location picking uses the awesome API from <a href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>. Opendata about current roadworks are imported from Lille, Lyon, Nancy, the Département de Seine-Saint-Denis, Sicoval and Versailles (all available under Etalab Licence Ouverte) as well as from Montpellier, Paris and Rennes (all available under ODbL license) and from the Département du Loiret. All opendata sources are fetched daily. More info about the licenses can be found in <a href=\"https://framagit.org/phyks/cyclassist/blob/master/README.md#license\">the README</a>.",
|
||||
"nbActiveReports": "No active report. |\u00a0One active report. | {nbActiveReports} active reports.",
|
||||
"nbReports": "No report. | One report. | {nbReports} reports.",
|
||||
"stats": "Stats",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user