From 1c568b689503c97d257a9c873563b908638f334a Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 15:37:11 +0200 Subject: [PATCH 01/30] sauvegarde des fichiers dans un sous-dossier --- recup_donnees_OSM_Overpass.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index a923696..fc2bc3f 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -20,9 +20,11 @@ import json import time from pyexcel_ods3 import save_data from collections import OrderedDict +import os overpass_url="http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" +dossier_sauvegarde = "resultats/" # nombre maxi de retries quand echec API max_retry = 4 @@ -437,7 +439,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # print (json.dumps(export_json)) - jsonFile = open(nom_req+".json", "w") + jsonFile = open(dossier_sauvegarde + nom_req+".json", "w") jsonFile.write(json.dumps(export_json)) jsonFile.close() @@ -478,7 +480,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, ODSdataSheet.update({"resultats": ODSdata}) - save_data(nom_req+".ods", ODSdataSheet) + save_data(dossier_sauvegarde + nom_req+".ods", ODSdataSheet) From 90201923becb9dd0f3ab86f531f4c8c33dff07bd Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 15:40:29 +0200 Subject: [PATCH 02/30] =?UTF-8?q?extraction=20des=20erreurs=20dans=20une?= =?UTF-8?q?=20classe=20s=C3=A9par=C3=A9e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/errors.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 osm_vc63/errors.py diff --git a/osm_vc63/errors.py b/osm_vc63/errors.py new file mode 100644 index 0000000..11dd585 --- /dev/null +++ b/osm_vc63/errors.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + + +class Api_error(Exception): + def __init__(self, http_code, message="erreur appel API"): + self.http_code = http_code + self.message = message + super().__init__(self.message) + + def __str__(self): + return f"{self.http_code} -> {self.message}" + + +class Overpass_error(Api_error): + pass + + +class Geo_api_error(Api_error): + pass From 769c0b8cdfc68e98bd51b306deaade2286f2e70c Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 15:42:12 +0200 Subject: [PATCH 03/30] import des erreurs dans le script principal --- recup_donnees_OSM_Overpass.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index fc2bc3f..afbb958 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -21,6 +21,7 @@ import time from pyexcel_ods3 import save_data from collections import OrderedDict import os +from osm_vc63 import errors overpass_url="http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" @@ -279,29 +280,12 @@ trad_bicycle_parking = { # ---------------------------------------------- -class Api_error(Exception) : - def __init__(self, http_code, message="erreur appel API"): - self.http_code = http_code - self.message = message - super().__init__(self.message) - - def __str__(self): - return f'{self.http_code} -> {self.message}' - -class Overpass_error(Api_error) : - pass - -class Geo_api_error(Api_error) : - pass - -# ---------------------------------------------- - def run_overpass_query(query) : response = requests.get(overpass_url, params={'data': query}) if (response.status_code != 200) : - raise Overpass_error(response.status_code) + raise errors.Overpass_error(response.status_code) return (response.json()) @@ -313,7 +297,7 @@ def run_reverse_geocoding(lat, lon) : response = requests.get(url, params={'lon' : str(lon), 'lat' : str(lat)}) if (response.status_code != 200) : - raise Geo_api_error(response.status_code) + raise errors.Geo_api_error(response.status_code) return (response.json()) @@ -495,7 +479,7 @@ for nom_requete in requetes_overpass.keys() : break - except Api_error : + except errors.Api_error : if (nb_essai == max_retry) : print ("trop d'erreurs d'API - abandon") From b2b0ee05513984675cc4db1263ffa15baf376c44 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 15:45:21 +0200 Subject: [PATCH 04/30] suppression de doublon --- recup_donnees_OSM_Overpass.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index afbb958..0ede27e 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -71,7 +71,6 @@ requetes_overpass = { "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, "access" : {"export_json" : "Oui", "FR" : "accès"}, - "amenity" : {"export_json" : "Non", "FR" : "aménagement"}, "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, "covered" : {"export_json" : "Oui", "FR" : "couvert"}, "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, @@ -90,7 +89,6 @@ requetes_overpass = { "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, "access" : {"export_json" : "Oui", "FR" : "accès"}, - "amenity" : {"export_json" : "Non", "FR" : "aménagement"}, "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, "covered" : {"export_json" : "Oui", "FR" : "couvert"}, "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, From 9b9b0ffd0dca04d446137698c540892317731f3e Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 18:09:13 +0200 Subject: [PATCH 05/30] =?UTF-8?q?gestion=20de=20la=20cr=C3=A9ation/existen?= =?UTF-8?q?ce=20du=20dossier=20de=20sauvegarde?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 0ede27e..97d62cb 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -420,6 +420,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # print (json.dumps(export_json)) + os.makedirs(dossier_sauvegarde, exist_ok = True) jsonFile = open(dossier_sauvegarde + nom_req+".json", "w") jsonFile.write(json.dumps(export_json)) From 8897764b9cc5d50bc1e7961645da9910de78e67c Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 3 Oct 2021 18:10:19 +0200 Subject: [PATCH 06/30] =?UTF-8?q?extraction=20des=20requ=C3=AAtes=20dans?= =?UTF-8?q?=20une=20classe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/requetes.py | 143 ++++++++++++++++++++++ recup_donnees_OSM_Overpass.py | 216 +--------------------------------- 2 files changed, 147 insertions(+), 212 deletions(-) create mode 100644 osm_vc63/requetes.py diff --git a/osm_vc63/requetes.py b/osm_vc63/requetes.py new file mode 100644 index 0000000..a8b9a9c --- /dev/null +++ b/osm_vc63/requetes.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + + +class requete: + nom: str + critere: str + champs: dict + + def __init__(self, nom, critere, champs): + self.nom = nom + self.critere = critere + self.champs = champs + + +reqs = [] +champs_stationnement = { + "amenity": {"export_json": "Non", "FR": "aménagement"}, + "capacity": {"export_json": "Oui", "FR": "nombre d'emplacements"}, + "access": {"export_json": "Oui", "FR": "accès"}, + "bicycle_parking": {"export_json": "Oui", "FR": "type"}, + "covered": {"export_json": "Oui", "FR": "couvert"}, + "operator": {"export_json": "Oui", "FR": "opérateur"}, + "operator:type": {"export_json": "Oui", "FR": "type d'opérateur"}, + "fee": {"export_json": "Oui", "FR": "frais"}, + "check_date:capacity": {"export_json": "Non", "FR": "date_vérification"}, + "source": {"export_json": "Non", "FR": "source"}, +} +champs_poi = { + "name": {"export_json": "Oui", "FR": ""}, + "description": {"export_json": "Oui", "FR": ""}, + "website": {"export_json": "Oui", "FR": ""}, + "addr:housenumber": {"export_json": "Oui", "FR": ""}, + "addr:street": {"export_json": "Oui", "FR": ""}, + "addr:postcode": {"export_json": "Oui", "FR": ""}, + "addr:city": {"export_json": "Oui", "FR": ""}, + "contact:email": {"export_json": "Oui", "FR": "email"}, + "contact:twitter": {"export_json": "Oui", "FR": "Twitter"}, + "contact:facebook": {"export_json": "Oui", "FR": "Facebook"}, + "contact:phone": {"export_json": "Oui", "FR": "Téléphone"}, + "network": {"export_json": "Oui", "FR": "Réseau"}, + "office": {"export_json": "Oui", "FR": "Bureau"}, + "opening_hours": {"export_json": "Oui", "FR": "Horaires"}, +} +# fields api_adresse (issus du géocodage inversé) +champs_adresse = { + "api_adresse:geometry:coordinates:lon": { + "export_json": "Non", + "FR": "lon_adresse_etalab", + }, + "api_adresse:geometry:coordinates:lat": { + "export_json": "Non", + "FR": "lat_adresse_etalab", + }, + "api_adresse:properties:label": {"export_json": "Non", "FR": "adresse_etalab"}, + "api_adresse:properties:score": {"export_json": "Non", "FR": "score_etalab"}, + "api_adresse:properties:housenumber": {"export_json": "Non", "FR": "numero_etalab"}, + "api_adresse:properties:type": {"export_json": "Non", "FR": "type_etalab"}, + "api_adresse:properties:name": { + "export_json": "Non", + "FR": "numero_et_voie_etalab", + }, + "api_adresse:properties:postcode": { + "export_json": "Non", + "FR": "code_postal_etalab", + }, + "api_adresse:properties:citycode": { + "export_json": "Non", + "FR": "code_INSEE_etalab", + }, + "api_adresse:properties:city": {"export_json": "Non", "FR": "ville_etalab"}, + "api_adresse:properties:street": {"export_json": "Non", "FR": "rue_etalab"}, +} + +reqs.append( + requete( + "stationnements_velos_publics", + r'nwr["amenity"="bicycle_parking"](area:aire_de_recherche); - nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', + dict(champs_stationnement, **champs_adresse), + ) +) + +reqs.append( + requete( + "stationnements_velos_non_publics", + r'nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', + dict(champs_stationnement, **champs_adresse), + ) +) + +champ_local = {"service:bicycle:diy": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "ateliers_autoreparation", + r'nwr["service:bicycle:diy"="yes"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + +champ_local = {"association": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "associations_velo", + r'nwr["association"="bicycle"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + +champ_local = {"craft": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "fabriquants_velo", + r'nwr["craft"="bicycle"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + +champ_local = {"shop": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "vendeurs_velo", + r'nwr["shop"="bicycle"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + +champ_local = {"amenity": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "velos_libre_service", + r'nwr["amenity"="bicycle_rental"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + +champ_local = {"service:bicycle:rental": {"export_json": "Non", "FR": ""}} +reqs.append( + requete( + "location_velo", + r'nwr["service:bicycle:rental"="yes"](area:aire_de_recherche);', + dict(champ_local, **champs_poi, **champs_adresse), + ) +) + diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 97d62cb..7fc310b 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -22,6 +22,7 @@ from pyexcel_ods3 import save_data from collections import OrderedDict import os from osm_vc63 import errors +from osm_vc63 import requetes overpass_url="http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" @@ -44,215 +45,6 @@ aire_de_recherche = str(3600000000+110866) -# fields api_adresse (issus du géocodage inversé) -api_adresse_fields = { "api_adresse:geometry:coordinates:lon" : {"export_json" : "Non", "FR" : "lon_adresse_etalab"}, - "api_adresse:geometry:coordinates:lat" : {"export_json" : "Non", "FR" : "lat_adresse_etalab"}, - "api_adresse:properties:label" : {"export_json" : "Non", "FR" : "adresse_etalab"}, - "api_adresse:properties:score" : {"export_json" : "Non", "FR" : "score_etalab"}, - "api_adresse:properties:housenumber" : {"export_json" : "Non", "FR" : "numero_etalab"}, - "api_adresse:properties:type" : {"export_json" : "Non", "FR" : "type_etalab"}, - "api_adresse:properties:name" : {"export_json" : "Non", "FR" : "numero_et_voie_etalab"}, - "api_adresse:properties:postcode" : {"export_json" : "Non", "FR" : "code_postal_etalab"}, - "api_adresse:properties:citycode" : {"export_json" : "Non", "FR" : "code_INSEE_etalab"}, - "api_adresse:properties:city" : {"export_json" : "Non", "FR" : "ville_etalab"}, - "api_adresse:properties:street" : {"export_json" : "Non", "FR" : "rue_etalab"}, - - } - - - -requetes_overpass = { - "stationnements_velos_publics" : - { - #"critere" : '["amenity"="bicycle_parking"]["access"~"(yes|public)"]', - "critere" : """nwr["amenity"="bicycle_parking"](area:"""+aire_de_recherche+"""); - - nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:"""+aire_de_recherche+");", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , - "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, - "access" : {"export_json" : "Oui", "FR" : "accès"}, - "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, - "covered" : {"export_json" : "Oui", "FR" : "couvert"}, - "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, - "operator:type" : {"export_json" : "Oui", "FR" : "type d'opérateur"}, - "fee" : {"export_json" : "Oui", "FR" : "frais"}, - "check_date:capacity" : {"export_json" : "Non", "FR" : "date_vérification"}, - "source" : {"export_json" : "Non", "FR" : "source"} - } - }, - - "stationnements_velos_non_publics" : - { - # "critere" : '["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers|permissive)"]', - "critere" : """nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:"""+aire_de_recherche+"); ", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : "aménagement"} , - "capacity" : {"export_json" : "Oui", "FR" : "nombre d'emplacements"}, - "access" : {"export_json" : "Oui", "FR" : "accès"}, - "bicycle_parking" : {"export_json" : "Oui", "FR" : "type"}, - "covered" : {"export_json" : "Oui", "FR" : "couvert"}, - "operator" : {"export_json" : "Oui", "FR" : "opérateur"}, - "operator:type" : {"export_json" : "Oui", "FR" : "type d'opérateur"}, - "fee" : {"export_json" : "Oui", "FR" : "frais"}, - "check_date:capacity" : {"export_json" : "Non", "FR" : "date_vérification"}, - "source" : {"export_json" : "Non", "FR" : "source"} - } - }, - - "ateliers_autoreparation" : - { - # "critere" : '["service:bicycle:diy"="yes"]', - "critere" : """nwr["service:bicycle:diy"="yes"](area:"""+aire_de_recherche+"); ", - "fields" : { - "service:bicycle:diy" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "associations_velo" : - { - #"critere" : '["association"="bicycle"]', - "critere" : """nwr["association"="bicycle"](area:"""+aire_de_recherche+"); ", - "fields" : { - "association" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "fabriquants_velo" : - { - #"critere" : '["craft"="bicycle"]', - "critere" : """nwr["craft"="bicycle"](area:"""+aire_de_recherche+"); ", - "fields" : { - "craft" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - - } - }, - - - "vendeurs_velo" : - { - #"critere" : '["shop"="bicycle"]', - "critere" : """nwr["shop"="bicycle"](area:"""+aire_de_recherche+"); ", - "fields" : { - "shop" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - "velos_libre_service" : - { - #"critere" : '["amenity"="bicycle_rental"]', - "critere" : """nwr["amenity"="bicycle_rental"](area:"""+aire_de_recherche+"); ", - "fields" : { - "amenity" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - }, - - - "location_velo" : - { - #"critere" : '["service:bicycle:rental"="yes"]', - "critere" : """nwr["service:bicycle:rental"="yes"](area:"""+aire_de_recherche+"); ", - "fields" : { - "service:bicycle:rental" : {"export_json" : "Non", "FR" : ""}, - "name" : {"export_json" : "Oui", "FR" : ""}, - "description" : {"export_json" : "Oui", "FR" : ""}, - "website" : {"export_json" : "Oui", "FR" : ""}, - "addr:housenumber" : {"export_json" : "Oui", "FR" : ""}, - "addr:street" : {"export_json" : "Oui", "FR" : ""}, - "addr:postcode" : {"export_json" : "Oui", "FR" : ""}, - "addr:city" : {"export_json" : "Oui", "FR" : ""}, - "contact:email" : {"export_json" : "Oui", "FR" : "email"}, - "contact:twitter" : {"export_json" : "Oui", "FR" : "Twitter"}, - "contact:facebook" : {"export_json" : "Oui", "FR" : "Facebook"}, - "contact:phone" : {"export_json" : "Oui", "FR" : "Téléphone"}, - "network" : {"export_json" : "Oui", "FR" : "Réseau"}, - "office" : {"export_json" : "Oui", "FR" : "Bureau"}, - "opening_hours" : {"export_json" : "Oui", "FR" : "Horaires"} - - } - } - - - } - - - - - # ---------------------------------------------- trad_bicycle_parking = { @@ -318,8 +110,8 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, out center; """ + overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) - overpass_query_fields.update(api_adresse_fields) print("Execution requete overpass : \n"+overpass_query) @@ -467,14 +259,14 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, -for nom_requete in requetes_overpass.keys() : +for req in requetes.reqs : for nb_essai in range(max_retry) : # on tente max_retry fois try : - executer_requete_et_exporter_resultats(nom_requete, requetes_overpass[nom_requete]["critere"], aire_de_recherche, requetes_overpass[nom_requete]["fields"]) + executer_requete_et_exporter_resultats(req.nom, req.critere, aire_de_recherche, req.champs) break From ab4af80601dd1e9d977ee73405b219e3fbe52b70 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 10:57:23 +0200 Subject: [PATCH 07/30] =?UTF-8?q?ajout=20des=20vendeurs=20de=20v=C3=A9los?= =?UTF-8?q?=20non=20sp=C3=A9cialis=C3=A9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/requetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm_vc63/requetes.py b/osm_vc63/requetes.py index a8b9a9c..ce6cf6d 100644 --- a/osm_vc63/requetes.py +++ b/osm_vc63/requetes.py @@ -118,7 +118,7 @@ champ_local = {"shop": {"export_json": "Non", "FR": ""}} reqs.append( requete( "vendeurs_velo", - r'nwr["shop"="bicycle"](area:aire_de_recherche);', + r'nwr["shop"="bicycle"](area:aire_de_recherche); + nwr["service:bicycle:retail"="yes"](area:aire_de_recherche)', dict(champ_local, **champs_poi, **champs_adresse), ) ) From e84075e0179f5d9b471515268d15d85895b8e687 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 11:35:06 +0200 Subject: [PATCH 08/30] =?UTF-8?q?extraction=20de=20la=20m=C3=A9thode=20de?= =?UTF-8?q?=20sauvegarde=20ods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 7fc310b..5774203 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -221,8 +221,10 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # =========================================== - print("Sauvegarde résultats format ODS") + sauvegarde_ods(overpass_query_fields, data, nom_req) + +def sauvegarde_ods(overpass_query_fields, data, nom_req): ODSdataSheet = OrderedDict() ODSdata = [] @@ -231,7 +233,6 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, index_line = 2 - for element in data["elements"]: line = [] @@ -239,14 +240,14 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, index_col = 0 # if (element["type"] == "node") : - for field in overpass_query_fields.keys() : - if (field in element["tags"]) : + for field in overpass_query_fields.keys(): + if field in element["tags"]: if field == "capacity": val = element["tags"][field] line.append(int(val) if val.isdigit() else val) - else : + else: line.append(element["tags"][field]) - else : + else: line.append("") index_col = index_col + 1 @@ -255,8 +256,9 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, ODSdataSheet.update({"resultats": ODSdata}) - save_data(dossier_sauvegarde + nom_req+".ods", ODSdataSheet) + save_data(dossier_sauvegarde + nom_req + ".ods", ODSdataSheet) + print("Sauvegarde résultats format ODS pour " + nom_req) for req in requetes.reqs : From 3a786f0f68c1ea2055af824e30f25ff05122c5a2 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 11:55:00 +0200 Subject: [PATCH 09/30] =?UTF-8?q?correction=20requ=C3=AAte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/requetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm_vc63/requetes.py b/osm_vc63/requetes.py index ce6cf6d..8bbbe00 100644 --- a/osm_vc63/requetes.py +++ b/osm_vc63/requetes.py @@ -118,7 +118,7 @@ champ_local = {"shop": {"export_json": "Non", "FR": ""}} reqs.append( requete( "vendeurs_velo", - r'nwr["shop"="bicycle"](area:aire_de_recherche); + nwr["service:bicycle:retail"="yes"](area:aire_de_recherche)', + r'nwr["shop"="bicycle"](area:aire_de_recherche); nwr["service:bicycle:retail"="yes"](area:aire_de_recherche);', dict(champ_local, **champs_poi, **champs_adresse), ) ) From eeac2f26aa8589d10baf918dd154ba3d06fb3676 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 14:31:01 +0200 Subject: [PATCH 10/30] =?UTF-8?q?cr=C3=A9ation=20du=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 5774203..89968dd 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -261,27 +261,29 @@ def sauvegarde_ods(overpass_query_fields, data, nom_req): print("Sauvegarde résultats format ODS pour " + nom_req) -for req in requetes.reqs : +def main(): + for req in requetes.reqs: - for nb_essai in range(max_retry) : + for nb_essai in range(max_retry): # on tente max_retry fois - try : - - executer_requete_et_exporter_resultats(req.nom, req.critere, aire_de_recherche, req.champs) - - break - - except errors.Api_error : + try: + executer_requete_et_exporter_resultats( + req.nom, req.critere, aire_de_recherche, req.champs + ) + break + except errors.Api_error: - if (nb_essai == max_retry) : - print ("trop d'erreurs d'API - abandon") + if nb_essai == max_retry: + print("trop d'erreurs d'API - abandon") exit() - print ("erreur API - on retente dans "+str(retry_delay)+"s") + print("erreur API - on retente dans " + str(retry_delay) + "s") time.sleep(retry_delay) + print("Fini") -print("Fini") +if __name__ == "__main__": + main() From 72135e0f3609daaf60fa21e864d2b28ac2031351 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 14:49:13 +0200 Subject: [PATCH 11/30] =?UTF-8?q?d=C3=A9placement=20et=20commentaire=20des?= =?UTF-8?q?=20m=C3=A9thodes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 84 ++++++++++++++++------------------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 89968dd..d1d230f 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -68,9 +68,42 @@ trad_bicycle_parking = { "floor": "Sol", "handlebar_holder": "Accroche-guidons"} -# ---------------------------------------------- + +def sauvegarde_ods(fields, data, dossier, nom_req): + """Sauvegarde de la requête""" + + ODSdataSheet = OrderedDict() + ODSdata = [] + ODSdata.append(fields.keys()) + index_line = 2 + + for element in data["elements"]: + line = [] + index_col = 0 + + for field in fields.keys(): + if field in element["tags"]: + if field == "capacity": + val = element["tags"][field] + line.append(int(val) if val.isdigit() else val) + else: + line.append(element["tags"][field]) + else: + line.append("") + index_col = index_col + 1 + + ODSdata.append(line) + index_line = index_line + 1 + + ODSdataSheet.update({"resultats": ODSdata}) + + save_data(dossier + nom_req + ".ods", ODSdataSheet) + + print("Sauvegarde résultats format ODS pour " + nom_req) + def run_overpass_query(query) : + """Envoie la requête Overpass et retourne la réponse JSON.""" response = requests.get(overpass_url, params={'data': query}) @@ -81,6 +114,7 @@ def run_overpass_query(query) : def run_reverse_geocoding(lat, lon) : + """Retourne une adresse JSON à partir d'une position GPS.""" url = geo_api_url + "/reverse/" @@ -91,14 +125,6 @@ def run_reverse_geocoding(lat, lon) : return (response.json()) -# ---------------------------------------------- - - - - - - - def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : print ("Nom requête : "+nom_req) @@ -221,45 +247,9 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # =========================================== - sauvegarde_ods(overpass_query_fields, data, nom_req) + sauvegarde_ods(overpass_query_fields, data, dossier_sauvegarde, nom_req) -def sauvegarde_ods(overpass_query_fields, data, nom_req): - ODSdataSheet = OrderedDict() - - ODSdata = [] - - ODSdata.append(overpass_query_fields.keys()) - - index_line = 2 - - for element in data["elements"]: - - line = [] - - index_col = 0 - - # if (element["type"] == "node") : - for field in overpass_query_fields.keys(): - if field in element["tags"]: - if field == "capacity": - val = element["tags"][field] - line.append(int(val) if val.isdigit() else val) - else: - line.append(element["tags"][field]) - else: - line.append("") - index_col = index_col + 1 - - ODSdata.append(line) - index_line = index_line + 1 - - ODSdataSheet.update({"resultats": ODSdata}) - - save_data(dossier_sauvegarde + nom_req + ".ods", ODSdataSheet) - - print("Sauvegarde résultats format ODS pour " + nom_req) - def main(): for req in requetes.reqs: @@ -287,3 +277,5 @@ def main(): if __name__ == "__main__": main() + + From feeb17d5924eb759a7040eb61b92cc3b231378cd Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 14:51:13 +0200 Subject: [PATCH 12/30] =?UTF-8?q?g=C3=A9ocodage=20invers=C3=A9=20comment?= =?UTF-8?q?=C3=A9=20pour=20all=C3=A9ger=20les=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index d1d230f..6f28976 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -146,7 +146,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, nb_elements = len(data["elements"]) print("Nombre d'elements : "+str(nb_elements)) - + """ print("Géocodage inversé : ", end="", flush=True) # @TODO : optimiser en faisant un appel au service /reverse/csv/ plutot que le service unitaire /reverse/ @@ -193,6 +193,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # print("-", end="", flush=True) print() + """ print("Sauvegarde résultat format JSON/OSM") From 6ba65776618e4e8c62ec33593fc725fe97a39500 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 14:56:40 +0200 Subject: [PATCH 13/30] commentaire --- recup_donnees_OSM_Overpass.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 6f28976..0f8c686 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -255,9 +255,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, def main(): for req in requetes.reqs: - for nb_essai in range(max_retry): - # on tente max_retry fois - + for nb_essai in range(max_retry): # on tente max_retry fois try: executer_requete_et_exporter_resultats( req.nom, req.critere, aire_de_recherche, req.champs From a78342fe73595a5a7e1da1cdfad290c777f6bdff Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 15:10:36 +0200 Subject: [PATCH 14/30] =?UTF-8?q?extraction=20de=20m=C3=A9thode=20sauvegar?= =?UTF-8?q?de=20json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 0f8c686..8bb246c 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -102,6 +102,12 @@ def sauvegarde_ods(fields, data, dossier, nom_req): print("Sauvegarde résultats format ODS pour " + nom_req) +def sauvegarde_json(export_json, dossier, nom_req): + jsonFile = open(dossier + nom_req + ".json", "w") + jsonFile.write(json.dumps(export_json)) + jsonFile.close() + + def run_overpass_query(query) : """Envoie la requête Overpass et retourne la réponse JSON.""" @@ -125,6 +131,7 @@ def run_reverse_geocoding(lat, lon) : return (response.json()) + def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : print ("Nom requête : "+nom_req) @@ -238,16 +245,10 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, index_line = index_line + 1 - # print (json.dumps(export_json)) + # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok = True) - jsonFile = open(dossier_sauvegarde + nom_req+".json", "w") - jsonFile.write(json.dumps(export_json)) - jsonFile.close() - - - # =========================================== - + sauvegarde_json(export_json, dossier_sauvegarde, nom_req) sauvegarde_ods(overpass_query_fields, data, dossier_sauvegarde, nom_req) From f97ffdb6d930ff02690b460cc5507d1ec9f7d7e1 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 15:23:03 +0200 Subject: [PATCH 15/30] extraction du nettoyage du JSON pour export --- recup_donnees_OSM_Overpass.py | 82 +++++++++++++++++------------------ 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 8bb246c..56bacf3 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -107,6 +107,45 @@ def sauvegarde_json(export_json, dossier, nom_req): jsonFile.write(json.dumps(export_json)) jsonFile.close() + print("Sauvegarde résultat format JSON/OSM " + nom_req) + + +def nettoyage_json_pour_umap(data, overpass_query_fields): + export_json = {"version": data["version"], + "generator" : data["generator"] + " and ETALAB API", + "osm3s" : data["osm3s"], + "elements": [] + } + + index_line = 0 + + for element in data["elements"]: + export_json["elements"].append({"type" : element["type"], + "id" : element["id"]}) + + # positionnement des éléments + if (element["type"] == "node") : # noeuds + export_json["elements"][index_line]["lat"] = element["lat"] + export_json["elements"][index_line]["lon"] = element["lon"] + else : # ways et relations + export_json["elements"][index_line]["center"] = element["center"] + export_json["elements"][index_line]["nodes"] = element["nodes"] + + # filtrage des tags + description = "" + for tag in overpass_query_fields.keys() : + if overpass_query_fields[tag]["export_json"] == "Oui" : + if tag in element["tags"] : + if overpass_query_fields[tag]["FR"] != "" : + description = description + overpass_query_fields[tag]["FR"] + " : " + + description = description + str(element["tags"][tag]) + "\n" + export_json["elements"][index_line]["tags"] = {"description": description} + + index_line = index_line + 1 + + return export_json + def run_overpass_query(query) : """Envoie la requête Overpass et retourne la réponse JSON.""" @@ -202,48 +241,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, print() """ - print("Sauvegarde résultat format JSON/OSM") - - export_json = {"version": data["version"], - "generator" : data["generator"] + " and ETALAB API", - "osm3s" : data["osm3s"], - "elements": [] - } - - index_line = 0 - - # on refait un JSON allégé juste avec les données qu'on va afficher sur la carte UMAP - - for element in data["elements"]: - - export_json["elements"].append({"type" : element["type"], - "id" : element["id"]}) - - if (element["type"] == "node") : - export_json["elements"][index_line]["lat"] = element["lat"] - export_json["elements"][index_line]["lon"] = element["lon"] - else : - export_json["elements"][index_line]["center"] = element["center"] - export_json["elements"][index_line]["nodes"] = element["nodes"] - - #export_json["elements"][index_line]["tags"] = element["tags"] - - description = "" - - for tag in overpass_query_fields.keys() : - if overpass_query_fields[tag]["export_json"] == "Oui" : - if tag in element["tags"] : - if overpass_query_fields[tag]["FR"] != "" : - description = description + overpass_query_fields[tag]["FR"] + " : " - - description = description + str(element["tags"][tag]) + "\n" - - - export_json["elements"][index_line]["tags"] = {"description": description} - - - index_line = index_line + 1 - + export_json = nettoyage_json_pour_umap(data, overpass_query_fields) # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok = True) From 27aea631bf124dde808296f6efbef6a6cdc7465d Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 15:28:41 +0200 Subject: [PATCH 16/30] =?UTF-8?q?d=C3=A9placement=20de=20logique=20dans=20?= =?UTF-8?q?la=20m=C3=A9thode=20de=20requ=C3=AAtage=20overpass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_OSM_Overpass.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 56bacf3..27a2a8d 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -147,10 +147,19 @@ def nettoyage_json_pour_umap(data, overpass_query_fields): return export_json -def run_overpass_query(query) : +def run_overpass_query(critere, aire_de_recherche) : """Envoie la requête Overpass et retourne la réponse JSON.""" + + overpass_query = """[out:json]; + ( + """+critere+""" + ); + out center; + """ + overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) - response = requests.get(overpass_url, params={'data': query}) + print("Execution requete overpass : \n" + overpass_query) + response = requests.get(overpass_url, params={'data': overpass_query}) if (response.status_code != 200) : raise errors.Overpass_error(response.status_code) @@ -173,21 +182,7 @@ def run_reverse_geocoding(lat, lon) : def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : - print ("Nom requête : "+nom_req) - - overpass_query = """[out:json]; - ( - """+critere+""" - ); - out center; - """ - - overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) - - - print("Execution requete overpass : \n"+overpass_query) - - data = run_overpass_query(overpass_query) + data = run_overpass_query(critere, aire_de_recherche) nb_elements = len(data["elements"]) From bcb9b7e9b2b09400c0cb1f2702ed51df10f3d42c Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 20:40:50 +0200 Subject: [PATCH 17/30] =?UTF-8?q?m=C3=A9thodes=20de=20sauvegarde=20d=C3=A9?= =?UTF-8?q?plac=C3=A9es=20dans=20une=20classe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/save.py | 48 +++++++++++++++++++++++++++++++++++ recup_donnees_OSM_Overpass.py | 46 +++------------------------------ 2 files changed, 51 insertions(+), 43 deletions(-) create mode 100644 osm_vc63/save.py diff --git a/osm_vc63/save.py b/osm_vc63/save.py new file mode 100644 index 0000000..b4c5426 --- /dev/null +++ b/osm_vc63/save.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import json +from pyexcel_ods3 import save_data +from collections import OrderedDict + + +class Save: + def as_ods(self, fields, data, dossier, nom_req): + """Sauvegarde de data dans un classeur ods""" + + ODSdataSheet = OrderedDict() + ODSdata = [] + ODSdata.append(fields.keys()) + index_line = 2 + + for element in data["elements"]: + line = [] + index_col = 0 + + for field in fields.keys(): + if field in element["tags"]: + if field == "capacity": + val = element["tags"][field] + line.append(int(val) if val.isdigit() else val) + else: + line.append(element["tags"][field]) + else: + line.append("") + index_col = index_col + 1 + + ODSdata.append(line) + index_line = index_line + 1 + + ODSdataSheet.update({"resultats": ODSdata}) + + save_data(dossier + nom_req + ".ods", ODSdataSheet) + + print("Sauvegarde résultats format ODS pour " + nom_req) + + def as_json(self, export_json, dossier, nom_req): + """Enregistrement du JSON""" + + jsonFile = open(dossier + nom_req + ".json", "w") + jsonFile.write(json.dumps(export_json)) + jsonFile.close() + + print("Sauvegarde résultat format JSON/OSM " + nom_req) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 27a2a8d..a112edf 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -23,6 +23,7 @@ from collections import OrderedDict import os from osm_vc63 import errors from osm_vc63 import requetes +from osm_vc63.save import Save overpass_url="http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" @@ -69,47 +70,6 @@ trad_bicycle_parking = { "handlebar_holder": "Accroche-guidons"} -def sauvegarde_ods(fields, data, dossier, nom_req): - """Sauvegarde de la requête""" - - ODSdataSheet = OrderedDict() - ODSdata = [] - ODSdata.append(fields.keys()) - index_line = 2 - - for element in data["elements"]: - line = [] - index_col = 0 - - for field in fields.keys(): - if field in element["tags"]: - if field == "capacity": - val = element["tags"][field] - line.append(int(val) if val.isdigit() else val) - else: - line.append(element["tags"][field]) - else: - line.append("") - index_col = index_col + 1 - - ODSdata.append(line) - index_line = index_line + 1 - - ODSdataSheet.update({"resultats": ODSdata}) - - save_data(dossier + nom_req + ".ods", ODSdataSheet) - - print("Sauvegarde résultats format ODS pour " + nom_req) - - -def sauvegarde_json(export_json, dossier, nom_req): - jsonFile = open(dossier + nom_req + ".json", "w") - jsonFile.write(json.dumps(export_json)) - jsonFile.close() - - print("Sauvegarde résultat format JSON/OSM " + nom_req) - - def nettoyage_json_pour_umap(data, overpass_query_fields): export_json = {"version": data["version"], "generator" : data["generator"] + " and ETALAB API", @@ -241,8 +201,8 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok = True) - sauvegarde_json(export_json, dossier_sauvegarde, nom_req) - sauvegarde_ods(overpass_query_fields, data, dossier_sauvegarde, nom_req) + Save().as_json(export_json, dossier_sauvegarde, nom_req) + Save().as_ods(overpass_query_fields, data, dossier_sauvegarde, nom_req) From 85eb60333a514305f68c96f60d85136a2802a344 Mon Sep 17 00:00:00 2001 From: SebF Date: Sat, 9 Oct 2021 20:47:09 +0200 Subject: [PATCH 18/30] =?UTF-8?q?d=C3=A9placement=20de=20la=20m=C3=A9thode?= =?UTF-8?q?=20de=20nettoyage=20json=20avec=20les=20m=C3=A9thodes=20de=20sa?= =?UTF-8?q?uvegarde?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/save.py | 42 +++++++++++++++++++++++++++++++++++ recup_donnees_OSM_Overpass.py | 37 +----------------------------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/osm_vc63/save.py b/osm_vc63/save.py index b4c5426..7334f6c 100644 --- a/osm_vc63/save.py +++ b/osm_vc63/save.py @@ -46,3 +46,45 @@ class Save: jsonFile.close() print("Sauvegarde résultat format JSON/OSM " + nom_req) + + def nettoyage_json_pour_umap(self, data, overpass_query_fields): + """Sélection uniquement des champs export_json == oui""" + + export_json = { + "version": data["version"], + "generator": data["generator"] + " and ETALAB API", + "osm3s": data["osm3s"], + "elements": [], + } + + index_line = 0 + + for element in data["elements"]: + export_json["elements"].append( + {"type": element["type"], "id": element["id"]} + ) + + # positionnement des éléments + if element["type"] == "node": # noeuds + export_json["elements"][index_line]["lat"] = element["lat"] + export_json["elements"][index_line]["lon"] = element["lon"] + else: # ways et relations + export_json["elements"][index_line]["center"] = element["center"] + export_json["elements"][index_line]["nodes"] = element["nodes"] + + # filtrage des tags + description = "" + for tag in overpass_query_fields.keys(): + if overpass_query_fields[tag]["export_json"] == "Oui": + if tag in element["tags"]: + if overpass_query_fields[tag]["FR"] != "": + description = ( + description + overpass_query_fields[tag]["FR"] + " : " + ) + + description = description + str(element["tags"][tag]) + "\n" + export_json["elements"][index_line]["tags"] = {"description": description} + + index_line = index_line + 1 + + return export_json diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index a112edf..6fabed2 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -70,41 +70,6 @@ trad_bicycle_parking = { "handlebar_holder": "Accroche-guidons"} -def nettoyage_json_pour_umap(data, overpass_query_fields): - export_json = {"version": data["version"], - "generator" : data["generator"] + " and ETALAB API", - "osm3s" : data["osm3s"], - "elements": [] - } - - index_line = 0 - - for element in data["elements"]: - export_json["elements"].append({"type" : element["type"], - "id" : element["id"]}) - - # positionnement des éléments - if (element["type"] == "node") : # noeuds - export_json["elements"][index_line]["lat"] = element["lat"] - export_json["elements"][index_line]["lon"] = element["lon"] - else : # ways et relations - export_json["elements"][index_line]["center"] = element["center"] - export_json["elements"][index_line]["nodes"] = element["nodes"] - - # filtrage des tags - description = "" - for tag in overpass_query_fields.keys() : - if overpass_query_fields[tag]["export_json"] == "Oui" : - if tag in element["tags"] : - if overpass_query_fields[tag]["FR"] != "" : - description = description + overpass_query_fields[tag]["FR"] + " : " - - description = description + str(element["tags"][tag]) + "\n" - export_json["elements"][index_line]["tags"] = {"description": description} - - index_line = index_line + 1 - - return export_json def run_overpass_query(critere, aire_de_recherche) : @@ -196,7 +161,7 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, print() """ - export_json = nettoyage_json_pour_umap(data, overpass_query_fields) + export_json = Save().nettoyage_json_pour_umap(data, overpass_query_fields) # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok = True) From 16ed9aa07b3df0e233c103e97db5920a562b1143 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 10:55:15 +0200 Subject: [PATCH 19/30] =?UTF-8?q?d=C3=A9placement=20des=20m=C3=A9thodes=20?= =?UTF-8?q?de=20requ=C3=AAte=20dans=20la=20classe=20Save=20renomm=C3=A9e?= =?UTF-8?q?=20en=20Utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/{save.py => utils.py} | 57 +++++++++++++++++++++++++++++++--- recup_donnees_OSM_Overpass.py | 41 ++++-------------------- 2 files changed, 58 insertions(+), 40 deletions(-) rename osm_vc63/{save.py => utils.py} (63%) diff --git a/osm_vc63/save.py b/osm_vc63/utils.py similarity index 63% rename from osm_vc63/save.py rename to osm_vc63/utils.py index 7334f6c..5943d8a 100644 --- a/osm_vc63/save.py +++ b/osm_vc63/utils.py @@ -1,12 +1,23 @@ #!/usr/bin/env python3 import json +import requests from pyexcel_ods3 import save_data from collections import OrderedDict +from osm_vc63 import errors -class Save: - def as_ods(self, fields, data, dossier, nom_req): +class Utils: + overpass_url: str + geo_api_url: str + dossier_sauvegarde: str + + def __init__(self, overpass_url, geo_api_url, dossier_sauvegarde): + self.overpass_url = overpass_url + self.geo_api_url = geo_api_url + self.dossier_sauvegarde = dossier_sauvegarde + + def as_ods(self, fields, data, nom_req): """Sauvegarde de data dans un classeur ods""" ODSdataSheet = OrderedDict() @@ -34,14 +45,14 @@ class Save: ODSdataSheet.update({"resultats": ODSdata}) - save_data(dossier + nom_req + ".ods", ODSdataSheet) + save_data(self.dossier_sauvegarde + nom_req + ".ods", ODSdataSheet) print("Sauvegarde résultats format ODS pour " + nom_req) - def as_json(self, export_json, dossier, nom_req): + def as_json(self, export_json, nom_req): """Enregistrement du JSON""" - jsonFile = open(dossier + nom_req + ".json", "w") + jsonFile = open(self.dossier_sauvegarde + nom_req + ".json", "w") jsonFile.write(json.dumps(export_json)) jsonFile.close() @@ -88,3 +99,39 @@ class Save: index_line = index_line + 1 return export_json + + def run_overpass_query(self, critere, aire_de_recherche): + """Envoie la requête Overpass et retourne la réponse JSON.""" + + overpass_query = ( + """[out:json]; + ( + """ + + critere + + """ + ); + out center; + """ + ) + overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) + + print("Execution requete overpass : \n" + overpass_query) + response = requests.get(self.overpass_url, params={"data": overpass_query}) + + if response.status_code != 200: + raise errors.Overpass_error(response.status_code) + + return response.json() + + def run_reverse_geocoding(self, lat, lon): + """Retourne une adresse JSON à partir d'une position GPS.""" + + url = self.geo_api_url + "/reverse/" + + response = requests.get(url, params={"lon": str(lon), "lat": str(lat)}) + + if response.status_code != 200: + raise errors.Geo_api_error(response.status_code) + + return response.json() + diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 6fabed2..49fb007 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -23,7 +23,7 @@ from collections import OrderedDict import os from osm_vc63 import errors from osm_vc63 import requetes -from osm_vc63.save import Save +from osm_vc63.utils import Utils overpass_url="http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" @@ -72,42 +72,13 @@ trad_bicycle_parking = { -def run_overpass_query(critere, aire_de_recherche) : - """Envoie la requête Overpass et retourne la réponse JSON.""" - - overpass_query = """[out:json]; - ( - """+critere+""" - ); - out center; - """ - overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) - print("Execution requete overpass : \n" + overpass_query) - response = requests.get(overpass_url, params={'data': overpass_query}) - - if (response.status_code != 200) : - raise errors.Overpass_error(response.status_code) - - return (response.json()) - - -def run_reverse_geocoding(lat, lon) : - """Retourne une adresse JSON à partir d'une position GPS.""" - - url = geo_api_url + "/reverse/" - - response = requests.get(url, params={'lon' : str(lon), 'lat' : str(lat)}) - - if (response.status_code != 200) : - raise errors.Geo_api_error(response.status_code) - - return (response.json()) def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : - data = run_overpass_query(critere, aire_de_recherche) + utils = Utils(overpass_url, geo_api_url, dossier_sauvegarde) + data = utils.run_overpass_query(critere, aire_de_recherche) nb_elements = len(data["elements"]) @@ -161,13 +132,13 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, print() """ - export_json = Save().nettoyage_json_pour_umap(data, overpass_query_fields) + export_json = utils.nettoyage_json_pour_umap(data, overpass_query_fields) # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok = True) - Save().as_json(export_json, dossier_sauvegarde, nom_req) - Save().as_ods(overpass_query_fields, data, dossier_sauvegarde, nom_req) + utils.as_json(export_json, nom_req) + utils.as_ods(overpass_query_fields, data, nom_req) From 7cfe3d3327d813fcdf7778a5a9a079b65a2c889d Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 10:56:13 +0200 Subject: [PATCH 20/30] formattage avec Black --- recup_donnees_OSM_Overpass.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 49fb007..c77d34b 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -25,7 +25,7 @@ from osm_vc63 import errors from osm_vc63 import requetes from osm_vc63.utils import Utils -overpass_url="http://overpass-api.de/api/interpreter" +overpass_url = "http://overpass-api.de/api/interpreter" geo_api_url = "https://api-adresse.data.gouv.fr" dossier_sauvegarde = "resultats/" @@ -42,8 +42,7 @@ retry_delay = 120 # id Romagnat : 138269 # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM -aire_de_recherche = str(3600000000+110866) - +aire_de_recherche = str(3600000000 + 110866) # ---------------------------------------------- @@ -67,22 +66,20 @@ trad_bicycle_parking = { "rope": "Câble", "two-tier": "Deux étages", "floor": "Sol", - "handlebar_holder": "Accroche-guidons"} + "handlebar_holder": "Accroche-guidons", +} - - - - - -def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, overpass_query_fields) : +def executer_requete_et_exporter_resultats( + nom_req, critere, aire_de_recherche, overpass_query_fields +): utils = Utils(overpass_url, geo_api_url, dossier_sauvegarde) data = utils.run_overpass_query(critere, aire_de_recherche) nb_elements = len(data["elements"]) - print("Nombre d'elements : "+str(nb_elements)) + print("Nombre d'elements : " + str(nb_elements)) """ print("Géocodage inversé : ", end="", flush=True) @@ -135,36 +132,34 @@ def executer_requete_et_exporter_resultats(nom_req, critere, aire_de_recherche, export_json = utils.nettoyage_json_pour_umap(data, overpass_query_fields) # Sauvegarde - os.makedirs(dossier_sauvegarde, exist_ok = True) + os.makedirs(dossier_sauvegarde, exist_ok=True) utils.as_json(export_json, nom_req) utils.as_ods(overpass_query_fields, data, nom_req) - def main(): for req in requetes.reqs: - for nb_essai in range(max_retry): # on tente max_retry fois + for nb_essai in range(max_retry): # on tente max_retry fois try: executer_requete_et_exporter_resultats( req.nom, req.critere, aire_de_recherche, req.champs ) break except errors.Api_error: - + if nb_essai == max_retry: print("trop d'erreurs d'API - abandon") exit() - + print("erreur API - on retente dans " + str(retry_delay) + "s") - + time.sleep(retry_delay) - + print("Fini") if __name__ == "__main__": main() - From 5dfa477ca7238865250786751ad4954b5d7511b9 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 16:31:54 +0200 Subject: [PATCH 21/30] pylint sur les erreurs --- osm_vc63/errors.py | 13 ++++++++++--- osm_vc63/utils.py | 4 ++-- recup_donnees_OSM_Overpass.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/osm_vc63/errors.py b/osm_vc63/errors.py index 11dd585..07f40f7 100644 --- a/osm_vc63/errors.py +++ b/osm_vc63/errors.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 +"""Errors module""" -class Api_error(Exception): +class ApiError(Exception): + """Api exception""" + def __init__(self, http_code, message="erreur appel API"): self.http_code = http_code self.message = message @@ -11,9 +14,13 @@ class Api_error(Exception): return f"{self.http_code} -> {self.message}" -class Overpass_error(Api_error): +class OverpassError(ApiError): + """Overpass exception""" + pass -class Geo_api_error(Api_error): +class GeoApiError(ApiError): + """GeoApi exception""" + pass diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index 5943d8a..b4def3a 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -119,7 +119,7 @@ class Utils: response = requests.get(self.overpass_url, params={"data": overpass_query}) if response.status_code != 200: - raise errors.Overpass_error(response.status_code) + raise errors.OverpassError(response.status_code) return response.json() @@ -131,7 +131,7 @@ class Utils: response = requests.get(url, params={"lon": str(lon), "lat": str(lat)}) if response.status_code != 200: - raise errors.Geo_api_error(response.status_code) + raise errors.GeoApiError(response.status_code) return response.json() diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index c77d34b..5f46281 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -147,7 +147,7 @@ def main(): req.nom, req.critere, aire_de_recherche, req.champs ) break - except errors.Api_error: + except errors.ApiError: if nb_essai == max_retry: print("trop d'erreurs d'API - abandon") From 29d2e602994f984e6d57c3cdd42e337c965e6218 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 16:32:16 +0200 Subject: [PATCH 22/30] =?UTF-8?q?renommage=20des=20m=C3=A9thodes=20de=20sa?= =?UTF-8?q?uvegarde?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/utils.py | 4 ++-- recup_donnees_OSM_Overpass.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index b4def3a..d754abe 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -17,7 +17,7 @@ class Utils: self.geo_api_url = geo_api_url self.dossier_sauvegarde = dossier_sauvegarde - def as_ods(self, fields, data, nom_req): + def save_as_ods(self, fields, data, nom_req): """Sauvegarde de data dans un classeur ods""" ODSdataSheet = OrderedDict() @@ -49,7 +49,7 @@ class Utils: print("Sauvegarde résultats format ODS pour " + nom_req) - def as_json(self, export_json, nom_req): + def save_as_json(self, export_json, nom_req): """Enregistrement du JSON""" jsonFile = open(self.dossier_sauvegarde + nom_req + ".json", "w") diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_OSM_Overpass.py index 5f46281..8c1dbca 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_OSM_Overpass.py @@ -134,8 +134,8 @@ def executer_requete_et_exporter_resultats( # Sauvegarde os.makedirs(dossier_sauvegarde, exist_ok=True) - utils.as_json(export_json, nom_req) - utils.as_ods(overpass_query_fields, data, nom_req) + utils.save_as_json(export_json, nom_req) + utils.save_as_ods(overpass_query_fields, data, nom_req) def main(): From 1c535dd9b9cc2a850feca58c9770ca14f9e7a647 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 16:35:27 +0200 Subject: [PATCH 23/30] pylint sur utils --- osm_vc63/utils.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index d754abe..bd61806 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +"""Collections de méthodes utilitaires""" import json +from collections import OrderedDict import requests from pyexcel_ods3 import save_data -from collections import OrderedDict from osm_vc63 import errors class Utils: + """Classe de méthodes utilitaires""" + overpass_url: str geo_api_url: str dossier_sauvegarde: str @@ -20,9 +23,9 @@ class Utils: def save_as_ods(self, fields, data, nom_req): """Sauvegarde de data dans un classeur ods""" - ODSdataSheet = OrderedDict() - ODSdata = [] - ODSdata.append(fields.keys()) + ods_data_sheet = OrderedDict() + ods_data = [] + ods_data.append(fields.keys()) index_line = 2 for element in data["elements"]: @@ -40,21 +43,21 @@ class Utils: line.append("") index_col = index_col + 1 - ODSdata.append(line) + ods_data.append(line) index_line = index_line + 1 - ODSdataSheet.update({"resultats": ODSdata}) + ods_data_sheet.update({"resultats": ods_data}) - save_data(self.dossier_sauvegarde + nom_req + ".ods", ODSdataSheet) + save_data(self.dossier_sauvegarde + nom_req + ".ods", ods_data_sheet) print("Sauvegarde résultats format ODS pour " + nom_req) def save_as_json(self, export_json, nom_req): """Enregistrement du JSON""" - jsonFile = open(self.dossier_sauvegarde + nom_req + ".json", "w") - jsonFile.write(json.dumps(export_json)) - jsonFile.close() + json_file = open(self.dossier_sauvegarde + nom_req + ".json", "w") + json_file.write(json.dumps(export_json)) + json_file.close() print("Sauvegarde résultat format JSON/OSM " + nom_req) @@ -134,4 +137,3 @@ class Utils: raise errors.GeoApiError(response.status_code) return response.json() - From af09edea2bf41e395426a1046b464cf6efd2ef92 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 16:50:27 +0200 Subject: [PATCH 24/30] pylint sur fichier principal --- ...erpass.py => recup_donnees_osm_overpass.py | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) rename recup_donnees_OSM_Overpass.py => recup_donnees_osm_overpass.py (81%) diff --git a/recup_donnees_OSM_Overpass.py b/recup_donnees_osm_overpass.py similarity index 81% rename from recup_donnees_OSM_Overpass.py rename to recup_donnees_osm_overpass.py index 8c1dbca..2a5dc59 100644 --- a/recup_donnees_OSM_Overpass.py +++ b/recup_donnees_osm_overpass.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +""" +Module principal :  +- récupération de données par appel à Overpass +- géocodage inverse +- export des données en JSON pour utilisation avec umap +- sauvegarde des données en ods +""" # inspiration : # https://towardsdatascience.com/loading-data-from-openstreetmap-with-python-and-the-overpass-api-513882a27fd0 @@ -15,25 +22,21 @@ # https://pythonhosted.org/pyexcel-ods/ # pip3 install pyexcel-ods3 -import requests -import json import time -from pyexcel_ods3 import save_data -from collections import OrderedDict import os from osm_vc63 import errors from osm_vc63 import requetes from osm_vc63.utils import Utils -overpass_url = "http://overpass-api.de/api/interpreter" -geo_api_url = "https://api-adresse.data.gouv.fr" -dossier_sauvegarde = "resultats/" +OVERPASS_URL = "http://overpass-api.de/api/interpreter" +GEO_API_URL = "https://api-adresse.data.gouv.fr" +DOSSIER_SAUVEGARDE = "resultats/" # nombre maxi de retries quand echec API -max_retry = 4 +MAX_RETRY = 4 # delai en secondes entre les tentatives -retry_delay = 120 +RETRY_DELAY = 120 # id du département "Puy de Dôme" : 7406 @@ -42,7 +45,7 @@ retry_delay = 120 # id Romagnat : 138269 # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM -aire_de_recherche = str(3600000000 + 110866) +AIRE_DE_RECHERCHE = str(3600000000 + 110866) # ---------------------------------------------- @@ -73,8 +76,16 @@ trad_bicycle_parking = { def executer_requete_et_exporter_resultats( nom_req, critere, aire_de_recherche, overpass_query_fields ): + """ + Appelle Overpass et exporte les résultats - utils = Utils(overpass_url, geo_api_url, dossier_sauvegarde) + nom_req : nom de la requête (type d'informations recherchées) + critere : requête passée à Overpass + aire_de_recherche : zone géographique d'intérêt + overpass_query_fields : champs récupérés pour la réponse + """ + + utils = Utils(OVERPASS_URL, GEO_API_URL, DOSSIER_SAUVEGARDE) data = utils.run_overpass_query(critere, aire_de_recherche) nb_elements = len(data["elements"]) @@ -132,34 +143,34 @@ def executer_requete_et_exporter_resultats( export_json = utils.nettoyage_json_pour_umap(data, overpass_query_fields) # Sauvegarde - os.makedirs(dossier_sauvegarde, exist_ok=True) + os.makedirs(DOSSIER_SAUVEGARDE, exist_ok=True) utils.save_as_json(export_json, nom_req) utils.save_as_ods(overpass_query_fields, data, nom_req) def main(): - for req in requetes.reqs: + """Routine principale""" - for nb_essai in range(max_retry): # on tente max_retry fois + for req in requetes.reqs: + for nb_essai in range(MAX_RETRY): # on tente max_retry fois try: executer_requete_et_exporter_resultats( - req.nom, req.critere, aire_de_recherche, req.champs + req.nom, req.critere, AIRE_DE_RECHERCHE, req.champs ) break except errors.ApiError: - if nb_essai == max_retry: + if nb_essai == MAX_RETRY: print("trop d'erreurs d'API - abandon") exit() - print("erreur API - on retente dans " + str(retry_delay) + "s") + print("erreur API - on retente dans " + str(RETRY_DELAY) + "s") - time.sleep(retry_delay) + time.sleep(RETRY_DELAY) print("Fini") if __name__ == "__main__": main() - From b5ab062b3e2472982f0f31e6dfd3d64bd128c1ae Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 16:53:12 +0200 Subject: [PATCH 25/30] pylint sur requetes --- osm_vc63/requetes.py | 75 ++++++++++++++++++----------------- recup_donnees_osm_overpass.py | 2 +- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/osm_vc63/requetes.py b/osm_vc63/requetes.py index 8bbbe00..ba3c29a 100644 --- a/osm_vc63/requetes.py +++ b/osm_vc63/requetes.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 +"""Module des requêtes""" -class requete: +class Requete: + """Objet requête""" + nom: str critere: str champs: dict @@ -12,8 +15,8 @@ class requete: self.champs = champs -reqs = [] -champs_stationnement = { +REQS = [] +CHAMPS_STATIONNEMENT = { "amenity": {"export_json": "Non", "FR": "aménagement"}, "capacity": {"export_json": "Oui", "FR": "nombre d'emplacements"}, "access": {"export_json": "Oui", "FR": "accès"}, @@ -25,7 +28,7 @@ champs_stationnement = { "check_date:capacity": {"export_json": "Non", "FR": "date_vérification"}, "source": {"export_json": "Non", "FR": "source"}, } -champs_poi = { +CHAMPS_POI = { "name": {"export_json": "Oui", "FR": ""}, "description": {"export_json": "Oui", "FR": ""}, "website": {"export_json": "Oui", "FR": ""}, @@ -41,8 +44,7 @@ champs_poi = { "office": {"export_json": "Oui", "FR": "Bureau"}, "opening_hours": {"export_json": "Oui", "FR": "Horaires"}, } -# fields api_adresse (issus du géocodage inversé) -champs_adresse = { +CHAMPS_ADRESSE = { "api_adresse:geometry:coordinates:lon": { "export_json": "Non", "FR": "lon_adresse_etalab", @@ -71,73 +73,72 @@ champs_adresse = { "api_adresse:properties:street": {"export_json": "Non", "FR": "rue_etalab"}, } -reqs.append( - requete( +REQS.append( + Requete( "stationnements_velos_publics", r'nwr["amenity"="bicycle_parking"](area:aire_de_recherche); - nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', - dict(champs_stationnement, **champs_adresse), + dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE), ) ) -reqs.append( - requete( +REQS.append( + Requete( "stationnements_velos_non_publics", r'nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', - dict(champs_stationnement, **champs_adresse), + dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE), ) ) -champ_local = {"service:bicycle:diy": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"service:bicycle:diy": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "ateliers_autoreparation", r'nwr["service:bicycle:diy"="yes"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) -champ_local = {"association": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"association": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "associations_velo", r'nwr["association"="bicycle"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) -champ_local = {"craft": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"craft": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "fabriquants_velo", r'nwr["craft"="bicycle"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) -champ_local = {"shop": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"shop": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "vendeurs_velo", r'nwr["shop"="bicycle"](area:aire_de_recherche); nwr["service:bicycle:retail"="yes"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) -champ_local = {"amenity": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"amenity": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "velos_libre_service", r'nwr["amenity"="bicycle_rental"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) -champ_local = {"service:bicycle:rental": {"export_json": "Non", "FR": ""}} -reqs.append( - requete( +CHAMP_LOCAL = {"service:bicycle:rental": {"export_json": "Non", "FR": ""}} +REQS.append( + Requete( "location_velo", r'nwr["service:bicycle:rental"="yes"](area:aire_de_recherche);', - dict(champ_local, **champs_poi, **champs_adresse), + dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) ) - diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index 2a5dc59..eed5029 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -152,7 +152,7 @@ def executer_requete_et_exporter_resultats( def main(): """Routine principale""" - for req in requetes.reqs: + for req in requetes.REQS: for nb_essai in range(MAX_RETRY): # on tente max_retry fois try: executer_requete_et_exporter_resultats( From 6d288c16725db0a481b1785b4f5c18834db888ce Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 17:55:26 +0200 Subject: [PATCH 26/30] =?UTF-8?q?extraction=20du=20g=C3=A9ocodage=20et=20t?= =?UTF-8?q?raduction=20vers=20utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- osm_vc63/utils.py | 73 +++++++++++++++++++++++- recup_donnees_osm_overpass.py | 104 +++++++--------------------------- 2 files changed, 92 insertions(+), 85 deletions(-) diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index bd61806..756ba23 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -118,7 +118,7 @@ class Utils: ) overpass_query = overpass_query.replace("aire_de_recherche", aire_de_recherche) - print("Execution requete overpass : \n" + overpass_query) + # print("Execution requete overpass : \n" + overpass_query) response = requests.get(self.overpass_url, params={"data": overpass_query}) if response.status_code != 200: @@ -137,3 +137,74 @@ class Utils: raise errors.GeoApiError(response.status_code) return response.json() + + # TODO : optimiser en faisant un appel au service /reverse/csv/ plutot que le service unitaire /reverse/ + def geocodage(self, data): + """Renseigne une adresse pour chaque élément de data""" + + for element in data["elements"]: + if element["type"] == "node": + rev_geocode = self.run_reverse_geocoding(element["lat"], element["lon"]) + else: + rev_geocode = self.run_reverse_geocoding( + element["center"]["lat"], element["center"]["lon"] + ) + + api_adresse = rev_geocode["features"][0] + + element["tags"]["api_adresse:geometry:coordinates:lon"] = api_adresse[ + "geometry" + ]["coordinates"][0] + element["tags"]["api_adresse:geometry:coordinates:lat"] = api_adresse[ + "geometry" + ]["coordinates"][1] + + element["tags"]["api_adresse:properties:label"] = api_adresse["properties"][ + "label" + ] + element["tags"]["api_adresse:properties:score"] = api_adresse["properties"][ + "score" + ] + + if "housenumber" in api_adresse["properties"]: + element["tags"]["api_adresse:properties:housenumber"] = api_adresse[ + "properties" + ]["housenumber"] + + element["tags"]["api_adresse:properties:type"] = api_adresse["properties"][ + "type" + ] + + element["tags"]["api_adresse:properties:name"] = api_adresse["properties"][ + "name" + ] + element["tags"]["api_adresse:properties:postcode"] = api_adresse[ + "properties" + ]["postcode"] + element["tags"]["api_adresse:properties:citycode"] = api_adresse[ + "properties" + ]["citycode"] + element["tags"]["api_adresse:properties:city"] = api_adresse["properties"][ + "city" + ] + + if "street" in api_adresse["properties"]: + element["tags"]["api_adresse:properties:street"] = api_adresse[ + "properties" + ]["street"] + + element["tags"]["api_adresse:properties:attribution"] = rev_geocode[ + "attribution" + ] + element["tags"]["api_adresse:properties:licence"] = rev_geocode["licence"] + + return data + + def traduction(self, tag, dictionnaire, data): + """Traduit le champ tag des éléments de data avec dict""" + + for element in data["elements"]: + if tag in element["tags"]: + element["tags"][tag] = dictionnaire[element["tags"][tag]] + + return data diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index eed5029..fc7b47e 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -43,14 +43,11 @@ RETRY_DELAY = 120 # id Riom : 1693144 # id Clermont : 110866 # id Romagnat : 138269 - # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM AIRE_DE_RECHERCHE = str(3600000000 + 110866) - -# ---------------------------------------------- - -trad_bicycle_parking = { +# traductions des tags bicycle_parking +TRAD_BICYCLE_PARKING = { "stands": "Arceaux", "wall_loops": "Pince roues", "rack": "Râteliers", @@ -73,91 +70,30 @@ trad_bicycle_parking = { } -def executer_requete_et_exporter_resultats( - nom_req, critere, aire_de_recherche, overpass_query_fields -): - """ - Appelle Overpass et exporte les résultats - - nom_req : nom de la requête (type d'informations recherchées) - critere : requête passée à Overpass - aire_de_recherche : zone géographique d'intérêt - overpass_query_fields : champs récupérés pour la réponse - """ - - utils = Utils(OVERPASS_URL, GEO_API_URL, DOSSIER_SAUVEGARDE) - data = utils.run_overpass_query(critere, aire_de_recherche) - - nb_elements = len(data["elements"]) - - print("Nombre d'elements : " + str(nb_elements)) - """ - print("Géocodage inversé : ", end="", flush=True) - - # @TODO : optimiser en faisant un appel au service /reverse/csv/ plutot que le service unitaire /reverse/ - - for element in data["elements"]: - - if (element["type"] == "node") : - rev_geocode = run_reverse_geocoding(element["lat"], element["lon"]) - else : - rev_geocode = run_reverse_geocoding(element["center"]["lat"], element["center"]["lon"]) - - api_adresse = rev_geocode["features"][0] - - element["tags"]["api_adresse:geometry:coordinates:lon"] = api_adresse["geometry"]["coordinates"][0] - element["tags"]["api_adresse:geometry:coordinates:lat"] = api_adresse["geometry"]["coordinates"][1] - - element["tags"]["api_adresse:properties:label"] = api_adresse["properties"]["label"] - element["tags"]["api_adresse:properties:score"] = api_adresse["properties"]["score"] - - if ("housenumber" in api_adresse["properties"]) : - element["tags"]["api_adresse:properties:housenumber"] = api_adresse["properties"]["housenumber"] - - element["tags"]["api_adresse:properties:type"] = api_adresse["properties"]["type"] - - element["tags"]["api_adresse:properties:name"] = api_adresse["properties"]["name"] - element["tags"]["api_adresse:properties:postcode"] = api_adresse["properties"]["postcode"] - element["tags"]["api_adresse:properties:citycode"] = api_adresse["properties"]["citycode"] - element["tags"]["api_adresse:properties:city"] = api_adresse["properties"]["city"] - - if ("street" in api_adresse["properties"]) : - element["tags"]["api_adresse:properties:street"] = api_adresse["properties"]["street"] - - element["tags"]["api_adresse:properties:attribution"] = rev_geocode["attribution"] - element["tags"]["api_adresse:properties:licence"] = rev_geocode["licence"] - - - # traduction - if "bicycle_parking" in element["tags"]: - element["tags"]["bicycle_parking"] = trad_bicycle_parking[element["tags"]["bicycle_parking"]] - - print("X", end="", flush=True) - - #else : - # print("-", end="", flush=True) - - print() - """ - - export_json = utils.nettoyage_json_pour_umap(data, overpass_query_fields) - - # Sauvegarde - os.makedirs(DOSSIER_SAUVEGARDE, exist_ok=True) - - utils.save_as_json(export_json, nom_req) - utils.save_as_ods(overpass_query_fields, data, nom_req) - - def main(): """Routine principale""" for req in requetes.REQS: for nb_essai in range(MAX_RETRY): # on tente max_retry fois try: - executer_requete_et_exporter_resultats( - req.nom, req.critere, AIRE_DE_RECHERCHE, req.champs - ) + utils = Utils(OVERPASS_URL, GEO_API_URL, DOSSIER_SAUVEGARDE) + + # appel overpass + data = utils.run_overpass_query(req.critere, AIRE_DE_RECHERCHE) + + # géocodage inverse + data = utils.geocodage(data) + + # traduction + data = utils.traduction("bicycle_parking", TRAD_BICYCLE_PARKING, data) + + # Sauvegarde + os.makedirs(DOSSIER_SAUVEGARDE, exist_ok=True) + export_json = utils.nettoyage_json_pour_umap(data, req.champs) + + utils.save_as_json(export_json, req.nom) + utils.save_as_ods(req.champs, data, req.nom) + break except errors.ApiError: From e6b432af4e3c20adb1b2cce562cb5fab2795226c Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 18:53:37 +0200 Subject: [PATCH 27/30] ajout d'une barre de progression --- osm_vc63/utils.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index 756ba23..5a2b54f 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -142,7 +142,8 @@ class Utils: def geocodage(self, data): """Renseigne une adresse pour chaque élément de data""" - for element in data["elements"]: + for element in self.progress_bar(data["elements"], prefix="Géocodage"): + if element["type"] == "node": rev_geocode = self.run_reverse_geocoding(element["lat"], element["lon"]) else: @@ -208,3 +209,43 @@ class Utils: element["tags"][tag] = dictionnaire[element["tags"][tag]] return data + + def progress_bar( + # pylint:disable=C0330 + self, + iterable, + decimals=1, + length=50, + prefix="", + fill="█", + print_end="\r", + ): + """ + Call in a loop to create terminal progress bar + @params: + iterable - Required : iterable object (Iterable) + decimals - Optional : positive number of decimals in percent complete (Int) + length - Optional : character length of bar (Int) + prefix - Optional : prefix string (Str) + fill - Optional : bar fill character (Str) + print_end - Optional : end character (e.g. "\r", "\r\n") (Str) + """ + total = len(iterable) + + if total == 0: + return + + # Initial Call + print(f"\r{prefix} |{'-' * length}| {0}%", end=print_end) + # Update Progress Bar + for i, item in enumerate(iterable): + yield item + percent = ("{0:." + str(decimals) + "f}").format( + 100 * ((i + 1) / float(total)) + ) + filled = int(length * (i + 1) // total) + progress = fill * filled + "-" * (length - filled) + print(f"\r{prefix} |{progress}| {percent}%", end=print_end) + + # Print New Line on Complete + print() From 4d05328ea3025cbd58a8a9899c27f0dd24610751 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 18:54:19 +0200 Subject: [PATCH 28/30] pylint et print de log --- osm_vc63/requetes.py | 3 +++ osm_vc63/utils.py | 4 ++-- recup_donnees_osm_overpass.py | 11 ++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/osm_vc63/requetes.py b/osm_vc63/requetes.py index ba3c29a..667cf8d 100644 --- a/osm_vc63/requetes.py +++ b/osm_vc63/requetes.py @@ -76,6 +76,7 @@ CHAMPS_ADRESSE = { REQS.append( Requete( "stationnements_velos_publics", + # pylint: disable=C0301 r'nwr["amenity"="bicycle_parking"](area:aire_de_recherche); - nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE), ) @@ -84,6 +85,7 @@ REQS.append( REQS.append( Requete( "stationnements_velos_non_publics", + # pylint: disable=C0301 r'nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers)"](area:aire_de_recherche);', dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE), ) @@ -120,6 +122,7 @@ CHAMP_LOCAL = {"shop": {"export_json": "Non", "FR": ""}} REQS.append( Requete( "vendeurs_velo", + # pylint: disable=C0301 r'nwr["shop"="bicycle"](area:aire_de_recherche); nwr["service:bicycle:retail"="yes"](area:aire_de_recherche);', dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE), ) diff --git a/osm_vc63/utils.py b/osm_vc63/utils.py index 5a2b54f..e767aef 100644 --- a/osm_vc63/utils.py +++ b/osm_vc63/utils.py @@ -50,7 +50,7 @@ class Utils: save_data(self.dossier_sauvegarde + nom_req + ".ods", ods_data_sheet) - print("Sauvegarde résultats format ODS pour " + nom_req) + print("Sauvegarde résultats format ODS") def save_as_json(self, export_json, nom_req): """Enregistrement du JSON""" @@ -59,7 +59,7 @@ class Utils: json_file.write(json.dumps(export_json)) json_file.close() - print("Sauvegarde résultat format JSON/OSM " + nom_req) + print("Sauvegarde résultat format JSON/OSM") def nettoyage_json_pour_umap(self, data, overpass_query_fields): """Sélection uniquement des champs export_json == oui""" diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index fc7b47e..9ae445d 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -13,9 +13,11 @@ Module principal :  # https://wiki.cartocite.fr/doku.php?id=umap:10_-_je_valorise_les_donnees_openstreetmap_avec_umap # https://sites-formations.univ-rennes2.fr/mastersigat/Cours/Intro_Overpass.pdf -# usage des tags : https://taginfo.openstreetmap.org/tags/?key=amenity&value=bicycle_parking#combinations +# usage des tags : +# https://taginfo.openstreetmap.org/tags/?key=amenity&value=bicycle_parking#combinations -# exemple URL données pour umap : https://www.velocite63.fr/velocite63/OSM/stationnements_velos_publics.json +# exemple URL données pour umap : +# https://www.velocite63.fr/velocite63/OSM/stationnements_velos_publics.json # penser à cocher "proxy" dans la rubrique "données distantes" du calque # export ODS : @@ -44,7 +46,7 @@ RETRY_DELAY = 120 # id Clermont : 110866 # id Romagnat : 138269 # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM -AIRE_DE_RECHERCHE = str(3600000000 + 110866) +AIRE_DE_RECHERCHE = str(3_600_000_000 + 110_866) # traductions des tags bicycle_parking TRAD_BICYCLE_PARKING = { @@ -78,8 +80,11 @@ def main(): try: utils = Utils(OVERPASS_URL, GEO_API_URL, DOSSIER_SAUVEGARDE) + print(f"{75*'#'}\r\nRequête en cours : {req.nom}") + # appel overpass data = utils.run_overpass_query(req.critere, AIRE_DE_RECHERCHE) + print(f"{len(data['elements'])} résultats") # géocodage inverse data = utils.geocodage(data) From 60078c92840f7a702cae7952d53ef8d370abf9f0 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 19:24:18 +0200 Subject: [PATCH 29/30] =?UTF-8?q?inutile=20de=20poursuivre=20s'il=20n'y=20?= =?UTF-8?q?a=20pas=20de=20r=C3=A9sultats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recup_donnees_osm_overpass.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index 9ae445d..68b6044 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -84,20 +84,24 @@ def main(): # appel overpass data = utils.run_overpass_query(req.critere, AIRE_DE_RECHERCHE) - print(f"{len(data['elements'])} résultats") + nb_resultats = len(data["elements"]) + print(f"{nb_resultats} résultats") - # géocodage inverse - data = utils.geocodage(data) + if nb_resultats > 0: + # géocodage inverse + data = utils.geocodage(data) - # traduction - data = utils.traduction("bicycle_parking", TRAD_BICYCLE_PARKING, data) + # traduction + data = utils.traduction( + "bicycle_parking", TRAD_BICYCLE_PARKING, data + ) - # Sauvegarde - os.makedirs(DOSSIER_SAUVEGARDE, exist_ok=True) - export_json = utils.nettoyage_json_pour_umap(data, req.champs) + # Sauvegarde + os.makedirs(DOSSIER_SAUVEGARDE, exist_ok=True) + export_json = utils.nettoyage_json_pour_umap(data, req.champs) - utils.save_as_json(export_json, req.nom) - utils.save_as_ods(req.champs, data, req.nom) + utils.save_as_json(export_json, req.nom) + utils.save_as_ods(req.champs, data, req.nom) break except errors.ApiError: From daf711de2c67f8ee3298590da928ac645c555604 Mon Sep 17 00:00:00 2001 From: SebF Date: Sun, 10 Oct 2021 19:53:43 +0200 Subject: [PATCH 30/30] temporisation sur Overpass --- recup_donnees_osm_overpass.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recup_donnees_osm_overpass.py b/recup_donnees_osm_overpass.py index 68b6044..acb5ab6 100644 --- a/recup_donnees_osm_overpass.py +++ b/recup_donnees_osm_overpass.py @@ -103,6 +103,8 @@ def main(): utils.save_as_json(export_json, req.nom) utils.save_as_ods(req.champs, data, req.nom) + # doucement sur overpass + time.sleep(30) break except errors.ApiError: @@ -114,7 +116,7 @@ def main(): time.sleep(RETRY_DELAY) - print("Fini") + print("\r\n ### Terminé ###") if __name__ == "__main__":