diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py deleted file mode 100644 index e6f8612..0000000 --- a/recup_donnees_OSM_Overpass.py +++ /dev/null @@ -1,509 +0,0 @@ -#!/usr/bin/env python3 - -# inspiration : -# https://towardsdatascience.com/loading-data-from-openstreetmap-with-python-and-the-overpass-api-513882a27fd0 -# https://geo.api.gouv.fr/adresse -# https://wiki.cartocite.fr/doku.php?id=umap:10_-_je_valorise_les_donnees_openstreetmap_avec_umap -# https://sites-formations.univ-rennes2.fr/mastersigat/Cours/Intro_Overpass.pdf - -# usage des tags : https://taginfo.openstreetmap.org/tags/?key=amenity&value=bicycle_parking#combinations - -# exemple URL données pour umap : https://www.velocite63.fr/velocite63/OSM/stationnements_velos_publics.json -# penser à cocher "proxy" dans la rubrique "données distantes" du calque - -# export ODS : -# https://pythonhosted.org/pyexcel-ods/ -# pip3 install pyexcel-ods3 - -import requests -import json -import time -from pyexcel_ods3 import save_data -from collections import OrderedDict - -overpass_url="http://overpass-api.de/api/interpreter" -geo_api_url = "https://api-adresse.data.gouv.fr" - -# nombre maxi de retries quand echec API -max_retry = 4 - -# delai en secondes entre les tentatives -retry_delay = 120 - - -# id du département "Puy de Dôme" : 7406 -# id Riom : 1693144 -# id Clermont : 110866 -# id Romagnat : 138269 - -# l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM -aire_de_recherche = str(3600000000+7406) - - - -# fields api_adresse (issus du géocodage inversé) -api_adresse_fields = { "api_adresse:geometry:coordinates:lon" : {"export_json" : "Non", "FR" : "lon_adresse_etalab"}, - "api_adresse:geometry:coordinates:lat" : {"export_json" : "Non", "FR" : "lat_adresse_etalab"}, - "api_adresse:properties:label" : {"export_json" : "Non", "FR" : "adresse_etalab"}, - "api_adresse:properties:score" : {"export_json" : "Non", "FR" : "score_etalab"}, - "api_adresse:properties:housenumber" : {"export_json" : "Non", "FR" : "numero_etalab"}, - "api_adresse:properties:type" : {"export_json" : "Non", "FR" : "type_etalab"}, - "api_adresse:properties:name" : {"export_json" : "Non", "FR" : "numero_et_voie_etalab"}, - "api_adresse:properties:postcode" : {"export_json" : "Non", "FR" : "code_postal_etalab"}, - "api_adresse:properties:citycode" : {"export_json" : "Non", "FR" : "code_INSEE_etalab"}, - "api_adresse:properties:city" : {"export_json" : "Non", "FR" : "ville_etalab"}, - "api_adresse:properties:street" : {"export_json" : "Non", "FR" : "rue_etalab"}, - - } - - - -requetes_overpass = { - "stationnements_velos_publics" : - { - #"critere" : '["amenity"="bicycle_parking"]["access"~"(yes|public)"]', - "critere" : """nwr["amenity"="bicycle_parking"](area:"""+aire_de_recherche+"""); - - nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:"""+aire_de_recherche+");", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , - "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, - "access" : {"export_json" : "Oui", "FR" : "accès"}, - "amenity" : {"export_json" : "Non", "FR" : "aménagement"}, - "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, - "covered" : {"export_json" : "Oui", "FR" : "couvert"}, - "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, - "operator:type" : {"export_json" : "Oui", "FR" : "type d'opérateur"}, - "fee" : {"export_json" : "Oui", "FR" : "frais"}, - "check_date:capacity" : {"export_json" : "Non", "FR" : "date_vérification"}, - "source" : {"export_json" : "Non", "FR" : "source"} - } - }, - - "stationnements_velos_non_publics" : - { - # "critere" : '["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers|permissive)"]', - "critere" : """nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:"""+aire_de_recherche+"); ", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , - "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, - "access" : {"export_json" : "Oui", "FR" : "accès"}, - "amenity" : {"export_json" : "Non", "FR" : "aménagement"}, - "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, - "covered" : {"export_json" : "Oui", "FR" : "couvert"}, - "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, - "operator:type" : {"export_json" : "Oui", "FR" : "type d'opérateur"}, - "fee" : {"export_json" : "Oui", "FR" : "frais"}, - "check_date:capacity" : {"export_json" : "Non", "FR" : "date_vérification"}, - "source" : {"export_json" : "Non", "FR" : "source"} - } - }, - - "ateliers_autoreparation" : - { - # "critere" : '["service:bicycle:diy"="yes"]', - "critere" : """nwr["service:bicycle:diy"="yes"](area:"""+aire_de_recherche+"); ", - "fields" : { - "service:bicycle:diy" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "associations_velo" : - { - #"critere" : '["association"="bicycle"]', - "critere" : """nwr["association"="bicycle"](area:"""+aire_de_recherche+"); ", - "fields" : { - "association" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "fabriquants_velo" : - { - #"critere" : '["craft"="bicycle"]', - "critere" : """nwr["craft"="bicycle"](area:"""+aire_de_recherche+"); ", - "fields" : { - "craft" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - - } - }, - - - "vendeurs_velo" : - { - #"critere" : '["shop"="bicycle"]', - "critere" : """nwr["shop"="bicycle"](area:"""+aire_de_recherche+"); "+ - """nwr["service:bicycle:retail"="yes"](area:"""+aire_de_recherche+"); ", - "fields" : { - "shop" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "velos_libre_service" : - { - #"critere" : '["amenity"="bicycle_rental"]', - "critere" : """nwr["amenity"="bicycle_rental"](area:"""+aire_de_recherche+"); ", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - - "location_velo" : - { - #"critere" : '["service:bicycle:rental"="yes"]', - "critere" : """nwr["service:bicycle:rental"="yes"](area:"""+aire_de_recherche+"); ", - "fields" : { - "service:bicycle:rental" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - } - - - } - - - - - -# ---------------------------------------------- - -trad_bicycle_parking = { - "stands": "Arceaux", - "wall_loops": "Pince roues", - "rack": "Râteliers", - "anchors": "Ancrage", - "shed": "Abri collectif", - "bollard": "Potelet", - "lockers": "Abris individuels", - "wide_stands": "Arceaux espacés", - "ground_slots": "Fente dans le sol", - "building": "Bâtiment", - "informal": "Informel", - "wave": "Râteliers", - "streetpod": "Arceaux", - "tree": "Arbre à bicyclettes", - "crossbar": "Barre", - "rope": "Câble", - "two-tier": "Deux étages", - "floor": "Sol", - "handlebar_holder": "Accroche-guidons"} - -# ---------------------------------------------- - -class Api_error(Exception) : - def __init__(self, http_code, message="erreur appel API"): - self.http_code = http_code - self.message = message - super().__init__(self.message) - - def __str__(self): - return f'{self.http_code} -> {self.message}' - -class Overpass_error(Api_error) : - pass - -class Geo_api_error(Api_error) : - pass - -# ---------------------------------------------- - -def run_overpass_query(query) : - - response = requests.get(overpass_url, params={'data': query}) - - if (response.status_code != 200) : - raise Overpass_error(response.status_code) - - return (response.json()) - - -def run_reverse_geocoding(lat, lon) : - - url = geo_api_url + "/reverse/" - - response = requests.get(url, params={'lon' : str(lon), 'lat' : str(lat)}) - - if (response.status_code != 200) : - raise Geo_api_error(response.status_code) - - return (response.json()) - -# ---------------------------------------------- - - - - - - - -def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : - - print ("Nom requête : "+nom_req) - - overpass_query = """[out:json]; - ( - """+critere+""" - ); - out center; - """ - - - overpass_query_fields.update(api_adresse_fields) - - print("Execution requete overpass : \n"+overpass_query) - - data = run_overpass_query(overpass_query) - - nb_elements = len(data["elements"]) - - print("Nombre d'elements : "+str(nb_elements)) - - print("Géocodage inversé : ", end="", flush=True) - - # @TODO : optimiser en faisant un appel au service /reverse/csv/ plutot que le service unitaire /reverse/ - - for element in data["elements"]: - - if (element["type"] == "node") : - rev_geocode = run_reverse_geocoding(element["lat"], element["lon"]) - else : - rev_geocode = run_reverse_geocoding(element["center"]["lat"], element["center"]["lon"]) - - api_adresse = rev_geocode["features"][0] - - element["tags"]["api_adresse:geometry:coordinates:lon"] = api_adresse["geometry"]["coordinates"][0] - element["tags"]["api_adresse:geometry:coordinates:lat"] = api_adresse["geometry"]["coordinates"][1] - - element["tags"]["api_adresse:properties:label"] = api_adresse["properties"]["label"] - element["tags"]["api_adresse:properties:score"] = api_adresse["properties"]["score"] - - if ("housenumber" in api_adresse["properties"]) : - element["tags"]["api_adresse:properties:housenumber"] = api_adresse["properties"]["housenumber"] - - element["tags"]["api_adresse:properties:type"] = api_adresse["properties"]["type"] - - element["tags"]["api_adresse:properties:name"] = api_adresse["properties"]["name"] - element["tags"]["api_adresse:properties:postcode"] = api_adresse["properties"]["postcode"] - element["tags"]["api_adresse:properties:citycode"] = api_adresse["properties"]["citycode"] - element["tags"]["api_adresse:properties:city"] = api_adresse["properties"]["city"] - - if ("street" in api_adresse["properties"]) : - element["tags"]["api_adresse:properties:street"] = api_adresse["properties"]["street"] - - element["tags"]["api_adresse:properties:attribution"] = rev_geocode["attribution"] - element["tags"]["api_adresse:properties:licence"] = rev_geocode["licence"] - - - # traduction - if "bicycle_parking" in element["tags"]: - element["tags"]["bicycle_parking"] = trad_bicycle_parking[element["tags"]["bicycle_parking"]] - - print("X", end="", flush=True) - - #else : - # print("-", end="", flush=True) - - print() - - print("Sauvegarde résultat format JSON/OSM") - - export_json = {"version": data["version"], - "generator" : data["generator"] + " and ETALAB API", - "osm3s" : data["osm3s"], - "elements": [] - } - - index_line = 0 - - # on refait un JSON allégé juste avec les données qu'on va afficher sur la carte UMAP - - for element in data["elements"]: - - export_json["elements"].append({"type" : element["type"], - "id" : element["id"]}) - - if (element["type"] == "node") : - export_json["elements"][index_line]["lat"] = element["lat"] - export_json["elements"][index_line]["lon"] = element["lon"] - else : - export_json["elements"][index_line]["center"] = element["center"] - export_json["elements"][index_line]["nodes"] = element["nodes"] - - #export_json["elements"][index_line]["tags"] = element["tags"] - - description = "" - - for tag in overpass_query_fields.keys() : - if overpass_query_fields[tag]["export_json"] == "Oui" : - if tag in element["tags"] : - if overpass_query_fields[tag]["FR"] != "" : - description = description + overpass_query_fields[tag]["FR"] + " : " - - description = description + str(element["tags"][tag]) + "\n" - - - export_json["elements"][index_line]["tags"] = {"description": description} - - - index_line = index_line + 1 - - - # print (json.dumps(export_json)) - - jsonFile = open(nom_req+".json", "w") - jsonFile.write(json.dumps(export_json)) - jsonFile.close() - - - # =========================================== - - print("Sauvegarde résultats format ODS") - - ODSdataSheet = OrderedDict() - - ODSdata = [] - - ODSdata.append(overpass_query_fields.keys()) - - index_line = 2 - - - for element in data["elements"]: - - line = [] - - index_col = 0 - - # if (element["type"] == "node") : - for field in overpass_query_fields.keys() : - if (field in element["tags"]) : - if field == "capacity": - val = element["tags"][field] - line.append(int(val) if val.isdigit() else val) - else : - line.append(element["tags"][field]) - else : - line.append("") - index_col = index_col + 1 - - ODSdata.append(line) - index_line = index_line + 1 - - ODSdataSheet.update({"resultats": ODSdata}) - - save_data(nom_req+".ods", ODSdataSheet) - - - -for nom_requete in requetes_overpass.keys() : - - for nb_essai in range(max_retry) : - # on tente max_retry fois - - try : - - executer_requete_et_exporter_resultats(nom_requete, requetes_overpass[nom_requete]["critere"], aire_de_recherche, requetes_overpass[nom_requete]["fields"]) - - break - - except Api_error : - - if (nb_essai == max_retry) : - print ("trop d'erreurs d'API - abandon") - exit() - - print ("erreur API - on retente dans "+str(retry_delay)+"s") - - time.sleep(retry_delay) - - - -print("Fini") diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index acb5ab6..d489b56 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -46,7 +46,7 @@ RETRY_DELAY = 120 # id Clermont : 110866 # id Romagnat : 138269 # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM -AIRE_DE_RECHERCHE = str(3_600_000_000 + 110_866) +AIRE_DE_RECHERCHE = str(3_600_000_000 + 7406) # traductions des tags bicycle_parking TRAD_BICYCLE_PARKING = {