Improve OpenData handling for roadwork, clear mention in About section and add OpenData for Loiret

This commit is contained in:
Lucas Verney 2018-10-18 15:03:33 +02:00
parent bbca4d88ea
commit 1e7b2b363a
3 changed files with 148 additions and 23 deletions

View File

@ -70,14 +70,16 @@ Icons are made from the original works:
Beep sound is provided under CC0 license and coming from Beep sound is provided under CC0 license and coming from
[freesound.org](https://freesound.org/people/thisusernameis/sounds/426888/). [freesound.org](https://freesound.org/people/thisusernameis/sounds/426888/).
The map background is using tiles from <a The map background is using tiles from
href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or [Carto.com](https://carto.com/location-data-services/basemaps/) or
<a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a [Thunderforest](http://thunderforest.com/), thanks to [OpenStreetMap
href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap contributors](https://www.openstreetmap.org/copyright) and
contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>. [Leaflet](http://leafletjs.com/).
Manual location picking uses the awesome API from <a Manual location picking uses the awesome API from [adresse.data.gouv.fr](https://adresse.data.gouv.fr).
href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>.
Details about the sources and licenses of the opendata imported is available
in the `scripts/opendata/` scripts.
## Thanks ## Thanks

View File

@ -21,8 +21,11 @@ from shapely.ops import transform
from server.models import db, Report from server.models import db, Report
level = logging.WARN level = logging.WARN
RAISE_ON_EXCEPT = False
if 'DEBUG' in os.environ: if 'DEBUG' in os.environ:
level = logging.info level = logging.INFO
if 'RAISE_ON_EXCEPT' in os.environ:
RAISE_ON_EXCEPT = True
logging.basicConfig(level=level) logging.basicConfig(level=level)
@ -40,6 +43,53 @@ def preprocess_lille(data):
return data return data
def preprocess_loiret(data):
out = []
for item in data['features']:
try:
if 'paths' in item['geometry']:
paths = item['geometry']['paths']
else:
paths = [
[
[item['geometry']['x'], item['geometry']['y']]
]
]
for path in paths:
if len(path) == 1:
geometry = {
'type': 'Point',
'coordinates': path[0]
}
else:
geometry = {
'type': 'LineString',
'coordinates': path
}
new_item = {
'fields': item['attributes'],
'geometry': geometry,
'recordid': item['attributes']['OBJECTID']
}
fields = new_item['fields']
# Homogeneize start and end date spelling
fields['date_debut'] = arrow.get(
float(fields['STARTDATE']) / 1000
).isoformat()
fields['date_fin'] = arrow.get(
float(fields['ENDDATE']) / 1000
).isoformat()
# Homogeneize geo shape
fields['geo_shape'] = geometry
out.append(new_item)
except KeyError as exc:
logging.warning('Invalid item in Loiret data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue
return out
def preprocess_lyon(data): def preprocess_lyon(data):
out = [] out = []
for item in data['features']: for item in data['features']:
@ -58,6 +108,8 @@ def preprocess_lyon(data):
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Lyon data: %s.', exc) logging.warning('Invalid item in Lyon data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -80,6 +132,8 @@ def preprocess_montpellier(data):
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Montpellier data: %s.', exc) logging.warning('Invalid item in Montpellier data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -113,6 +167,8 @@ def preprocess_nancy(data):
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Nancy data: %s.', exc) logging.warning('Invalid item in Nancy data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -134,6 +190,8 @@ def preprocess_rennes(data):
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Rennes data: %s.', exc) logging.warning('Invalid item in Rennes data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -153,6 +211,8 @@ def preprocess_seine_saint_denis(data):
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Seine-Saint-Denis data: %s.', exc) logging.warning('Invalid item in Seine-Saint-Denis data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -168,6 +228,8 @@ def preprocess_sicoval(data):
fields['geo_shape'] = fields['geoshape2'] fields['geo_shape'] = fields['geoshape2']
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Sicoval data: %s.', exc) logging.warning('Invalid item in Sicoval data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return data return data
@ -181,6 +243,8 @@ def preprocess_toulouse(data):
fields['date_fin'] = fields['datefin'] fields['date_fin'] = fields['datefin']
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Toulouse data: %s.', exc) logging.warning('Invalid item in Toulouse data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return data return data
@ -189,20 +253,45 @@ def preprocess_versailles(data):
out = [] out = []
for item in data['features']: for item in data['features']:
try: try:
if 'paths' in item['geometry']:
paths = item['geometry']['paths']
else:
paths = [
[
[item['geometry']['x'], item['geometry']['y']]
]
]
for path in paths:
if len(path) == 1:
geometry = {
'type': 'Point',
'coordinates': path[0]
}
else:
geometry = {
'type': 'LineString',
'coordinates': path
}
new_item = { new_item = {
'fields': item['properties'], 'fields': item['attributes'],
'geometry': item['geometry'], 'geometry': geometry,
'recordid': item['properties']['OBJECTID'] 'recordid': item['attributes']['OBJECTID']
} }
fields = new_item['fields'] fields = new_item['fields']
# Homogeneize start date and end date spelling # Homogeneize start and end date spelling
fields['date_debut'] = fields['STARTDATE'] fields['date_debut'] = arrow.get(
fields['date_fin'] = fields['ENDDATE'] float(fields['STARTDATE']) / 1000
).isoformat()
fields['date_fin'] = arrow.get(
float(fields['ENDDATE']) / 1000
).isoformat()
# Homogeneize geo shape # Homogeneize geo shape
fields['geo_shape'] = item['geometry'] fields['geo_shape'] = geometry
out.append(new_item) out.append(new_item)
except KeyError as exc: except KeyError as exc:
logging.warning('Invalid item in Versailles data: %s.', exc) logging.warning('Invalid item in Versailles data: %s.', exc)
if RAISE_ON_EXCEPT:
raise
continue continue
return out return out
@ -210,54 +299,88 @@ def preprocess_versailles(data):
MIN_DISTANCE_REPORT_DETAILS = 40 # Distance in meters, same as in constants.js MIN_DISTANCE_REPORT_DETAILS = 40 # Distance in meters, same as in constants.js
OPENDATA_URLS = { OPENDATA_URLS = {
# Work in Lille # Work in Lille
# https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/
# Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
"lille": { "lille": {
"preprocess": preprocess_lille, "preprocess": preprocess_lille,
"url": "https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/download/?format=json&timezone=Europe/Berlin" "url": "https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/download/?format=json&timezone=Europe/Berlin"
}, },
# Work in Loiret
# https://open-loiret.opendata.arcgis.com/datasets/74c95548589d4ddeb3fcf094f7d61a67_1?geometry=0.609%2C47.694%2C3.245%2C48.016&orderBy=BLOCKNM
# Custom license
"loiret": {
"preprocess": preprocess_loiret,
"url": "https://services2.arcgis.com/IEzPuQhCEVCtkVvT/arcgis/rest/services/Travaux_routiers/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson",
},
# Work in Lyon # Work in Lyon
# https://data.grandlyon.com/equipements/chantiers-perturbants-de-la-mftropole-de-lyon/
# Licence Ouverte : https://download.data.grandlyon.com/files/grandlyon/LicenceOuverte.pdf
"lyon": { "lyon": {
"preprocess": preprocess_lyon, "preprocess": preprocess_lyon,
"url": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0&outputformat=GEOJSON&maxfeatures=30&request=GetFeature&typename=pvo_patrimoine_voirie.pvochantierperturbant&SRSNAME=urn:ogc:def:crs:EPSG::4171", "url": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0&outputformat=GEOJSON&maxfeatures=30&request=GetFeature&typename=pvo_patrimoine_voirie.pvochantierperturbant&SRSNAME=urn:ogc:def:crs:EPSG::4171",
}, },
# Work in Montpellier # Work in Montpellier
# http://data.montpellier3m.fr/dataset/chantiers-genants-de-montpellier
# Licence ODbL : http://opendefinition.org/licenses/odc-odbl/
"montpellier": { "montpellier": {
"preprocess": preprocess_montpellier, "preprocess": preprocess_montpellier,
"url": "http://data.montpellier3m.fr/sites/default/files/ressources/MMM_MMM_Chantiers.json" "url": "http://data.montpellier3m.fr/sites/default/files/ressources/MMM_MMM_Chantiers.json"
}, },
# Work in Nancy # Work in Nancy
# http://opendata.grandnancy.eu/jeux-de-donnees/detail-dune-fiche-de-donnees/?tx_icsoddatastore_pi1%5Bkeywords%5D=travaux&tx_icsoddatastore_pi1%5Buid%5D=63&tx_icsoddatastore_pi1%5BreturnID%5D=447
# Licence libre / Licence Ouverte (Etalab)
"nancy": { "nancy": {
"preprocess": preprocess_nancy, "preprocess": preprocess_nancy,
"url": "https://geoservices.grand-nancy.org/arcgis/rest/services/public/VOIRIE_Info_Travaux_Niveau/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson" "url": "https://geoservices.grand-nancy.org/arcgis/rest/services/public/VOIRIE_Info_Travaux_Niveau/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
}, },
# Work in Paris # Work in Paris
# https://opendata.paris.fr/explore/dataset/chantiers-perturbants/
# Licence ODbL : http://opendatacommons.org/licenses/odbl/
"paris": { "paris": {
"preprocess": None, "preprocess": None,
"url": "https://opendata.paris.fr/explore/dataset/chantiers-perturbants/download/?format=json&timezone=Europe/Berlin", "url": "https://opendata.paris.fr/explore/dataset/chantiers-perturbants/download/?format=json&timezone=Europe/Berlin",
}, },
# Work in Rennes # Work in Rennes
# http://travaux.data.rennesmetropole.fr/
# Usage libre / Licence ODbL : http://opendatacommons.org/licenses/odbl/
"rennes": { "rennes": {
"preprocess": preprocess_rennes, "preprocess": preprocess_rennes,
"url": "http://travaux.data.rennesmetropole.fr/api/roadworks?epsg=4326" "url": "http://travaux.data.rennesmetropole.fr/api/roadworks?epsg=4326"
}, },
# Work in Seine-Saint-Denis # Work in Seine-Saint-Denis
# https://geo.data.gouv.fr/fr/datasets/12504debb9bb73e717ad710a746541ebf817d98c
# Licence Ouverte : https://www.etalab.gouv.fr/licence-ouverte-open-licence
"seine-saint-denis": { "seine-saint-denis": {
"preprocess": preprocess_seine_saint_denis, "preprocess": preprocess_seine_saint_denis,
"url": "https://geo.data.gouv.fr/api/geogw/services/5779810963f06a3a8e81541b/feature-types/C1296/download?format=GeoJSON&projection=WGS84" "url": "https://geo.data.gouv.fr/api/geogw/services/5779810963f06a3a8e81541b/feature-types/C1296/download?format=GeoJSON&projection=WGS84"
}, },
# Work in Sicoval (South of Toulouse) # Work in Sicoval (South of Toulouse)
# https://data.opendatasoft.com/explore/dataset/travauxincidents@sicoval-haute-garonne/
# Licence Ouverte v2.0 (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2017/04/ETALAB-Licence-Ouverte-v2.0.pdf
"sicoval": { "sicoval": {
"preprocess": preprocess_sicoval, "preprocess": preprocess_sicoval,
"url": "https://data.opendatasoft.com/explore/dataset/travauxincidents@sicoval-haute-garonne/download/?format=json&timezone=Europe/Berlin" "url": "https://data.opendatasoft.com/explore/dataset/travauxincidents@sicoval-haute-garonne/download/?format=json&timezone=Europe/Berlin"
}, },
# Work in Toulouse # Work in Toulouse
# https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/
# Licence ODbL : http://opendatacommons.org/licenses/odbl/
"toulouse": { "toulouse": {
"preprocess": preprocess_toulouse, "preprocess": preprocess_toulouse,
"url": "https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/download/?format=json&timezone=Europe/Berlin", "url": "https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/download/?format=json&timezone=Europe/Berlin",
}, },
# Work in Versailles # Work in Versailles
"versailles": { # Licence Ouverte (Etalab)
"versailles-blocks": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_0
"preprocess": preprocess_versailles, "preprocess": preprocess_versailles,
"url": "https://www.data.gouv.fr/en/datasets/r/16f90f14-7896-4463-bc35-9809b55c2c1b" "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
},
"versailles-closures": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_1
"preprocess": preprocess_versailles,
"url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
},
"versailles-detours": { # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_2
"preprocess": preprocess_versailles,
"url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
}, },
} }
REPORT_TYPE = 'interrupt' REPORT_TYPE = 'interrupt'

View File

@ -3,7 +3,7 @@
"availableReportsTitle": "The available reports so far are:", "availableReportsTitle": "The available reports so far are:",
"lastReportAdded": "Last report added {fromNow}.", "lastReportAdded": "Last report added {fromNow}.",
"license": "License", "license": "License",
"licenseDescription": "It is released under an <a href=\"https://opensource.org/licenses/MIT\">MIT license</a> (<a href=\"https://framagit.org/phyks/cyclassist\">source code</a>). Icons are based on creations from Wikimedia, Vecteezy, Pixabay or Flaticon. Sounds are based on CC0 works from <a href=\"https://freesound.org/\">freesound.org</a>. The map background is using tiles from <a href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or <a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>. Collected reports are available under <a href=\"https://opendatacommons.org/licenses/odbl/\">ODbL license</a>. Manual location picking uses the awesome API from <a href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>.", "licenseDescription": "It is released under an <a href=\"https://opensource.org/licenses/MIT\">MIT license</a> (<a href=\"https://framagit.org/phyks/cyclassist\">source code</a>). Icons are based on creations from Wikimedia, Vecteezy, Pixabay or Flaticon. Sounds are based on CC0 works from <a href=\"https://freesound.org/\">freesound.org</a>. The map background is using tiles from <a href=\"https://carto.com/location-data-services/basemaps/\">Carto.com</a> or <a href=\"http://thunderforest.com/\">Thunderforest</a>, thanks to <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap contributors</a> and <a href=\"http://leafletjs.com/\">Leaflet</a>. Collected reports are available under <a href=\"https://opendatacommons.org/licenses/odbl/\">ODbL license</a>. Manual location picking uses the awesome API from <a href=\"https://adresse.data.gouv.fr\">adresse.data.gouv.fr</a>. Opendata about current roadworks are imported from Lille, Lyon, Nancy, the Département de Seine-Saint-Denis, Sicoval and Versailles (all available under Etalab Licence Ouverte) as well as from Montpellier, Paris and Rennes (all available under ODbL license) and from the Département du Loiret. All opendata sources are fetched daily. More info about the licenses can be found in <a href=\"https://framagit.org/phyks/cyclassist/blob/master/README.md#license\">the README</a>.",
"nbActiveReports": "No active report. |\u00a0One active report. | {nbActiveReports} active reports.", "nbActiveReports": "No active report. |\u00a0One active report. | {nbActiveReports} active reports.",
"nbReports": "No report. | One report. | {nbReports} reports.", "nbReports": "No report. | One report. | {nbReports} reports.",
"stats": "Stats", "stats": "Stats",