2021-09-26 12:52:37 +02:00
|
|
|
|
#!/usr/bin/env python3
|
2021-10-21 22:27:19 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Copyright 2021 Olav63, SebF
|
|
|
|
|
#
|
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
|
# (at your option) any later version.
|
|
|
|
|
#
|
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
|
#
|
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
|
|
|
2021-10-10 16:50:27 +02:00
|
|
|
|
"""
|
|
|
|
|
Module principal :
|
|
|
|
|
- récupération de données par appel à Overpass
|
|
|
|
|
- géocodage inverse
|
|
|
|
|
- export des données en JSON pour utilisation avec umap
|
|
|
|
|
- sauvegarde des données en ods
|
|
|
|
|
"""
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import time
|
2021-10-03 15:37:11 +02:00
|
|
|
|
import os
|
2021-10-22 22:09:41 +02:00
|
|
|
|
import argparse
|
2021-10-23 19:48:52 +02:00
|
|
|
|
import logging
|
|
|
|
|
import sys
|
2021-10-03 15:42:12 +02:00
|
|
|
|
from osm_vc63 import errors
|
2021-10-03 18:10:19 +02:00
|
|
|
|
from osm_vc63 import requetes
|
2021-10-10 10:55:15 +02:00
|
|
|
|
from osm_vc63.utils import Utils
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 16:50:27 +02:00
|
|
|
|
OVERPASS_URL = "http://overpass-api.de/api/interpreter"
|
|
|
|
|
GEO_API_URL = "https://api-adresse.data.gouv.fr"
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
|
|
|
|
# nombre maxi de retries quand echec API
|
2021-10-10 16:50:27 +02:00
|
|
|
|
MAX_RETRY = 4
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
|
|
|
|
# delai en secondes entre les tentatives
|
2021-10-10 16:50:27 +02:00
|
|
|
|
RETRY_DELAY = 120
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
|
|
|
|
|
2021-10-10 17:55:26 +02:00
|
|
|
|
# traductions des tags bicycle_parking
|
|
|
|
|
TRAD_BICYCLE_PARKING = {
|
2021-09-26 12:52:37 +02:00
|
|
|
|
"stands": "Arceaux",
|
|
|
|
|
"wall_loops": "Pince roues",
|
2023-08-20 21:48:50 +02:00
|
|
|
|
"wall_hoops": "Pince roues",
|
2021-09-26 12:52:37 +02:00
|
|
|
|
"rack": "Râteliers",
|
|
|
|
|
"anchors": "Ancrage",
|
|
|
|
|
"shed": "Abri collectif",
|
|
|
|
|
"bollard": "Potelet",
|
|
|
|
|
"lockers": "Abris individuels",
|
|
|
|
|
"wide_stands": "Arceaux espacés",
|
|
|
|
|
"ground_slots": "Fente dans le sol",
|
|
|
|
|
"building": "Bâtiment",
|
|
|
|
|
"informal": "Informel",
|
|
|
|
|
"wave": "Râteliers",
|
|
|
|
|
"streetpod": "Arceaux",
|
|
|
|
|
"tree": "Arbre à bicyclettes",
|
|
|
|
|
"crossbar": "Barre",
|
|
|
|
|
"rope": "Câble",
|
|
|
|
|
"two-tier": "Deux étages",
|
|
|
|
|
"floor": "Sol",
|
2021-10-10 10:56:13 +02:00
|
|
|
|
"handlebar_holder": "Accroche-guidons",
|
|
|
|
|
}
|
2021-10-09 15:10:36 +02:00
|
|
|
|
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-22 22:09:41 +02:00
|
|
|
|
def init_argparse() -> argparse.ArgumentParser:
|
|
|
|
|
"""Définition des arguments possibles."""
|
|
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
|
usage="%(prog)s [OPTIONS] ...",
|
|
|
|
|
description="Exporte les données de la Cyclosphère d'une zone géographique.",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-z",
|
|
|
|
|
"--zone",
|
|
|
|
|
type=int,
|
2021-10-23 19:48:52 +02:00
|
|
|
|
help="Choisir la zone géographique à inspecter.",
|
2021-10-22 22:09:41 +02:00
|
|
|
|
default=7406,
|
|
|
|
|
)
|
|
|
|
|
|
2021-10-22 22:35:56 +02:00
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-gi",
|
|
|
|
|
"--geocoding-inverse",
|
|
|
|
|
dest="geocoding_inverse",
|
|
|
|
|
action="store_true",
|
|
|
|
|
help="Activer le géocodage inverse",
|
|
|
|
|
)
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-ngi",
|
|
|
|
|
"--no-geocoding-inverse",
|
|
|
|
|
dest="geocoding_inverse",
|
|
|
|
|
action="store_false",
|
|
|
|
|
help="Désactiver le géocodage inverse",
|
|
|
|
|
)
|
|
|
|
|
parser.set_defaults(geocoding_inverse=True)
|
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
parser.add_argument(
|
2021-10-23 22:11:11 +02:00
|
|
|
|
"-l", "--log-level", type=str, help="Définir le niveau de log.", default="INFO"
|
2021-10-23 19:48:52 +02:00
|
|
|
|
)
|
|
|
|
|
|
2021-10-24 08:24:41 +02:00
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-r",
|
|
|
|
|
"--dossier-resultats",
|
|
|
|
|
type=str,
|
|
|
|
|
help="Définir le dossier d'archive",
|
|
|
|
|
default="resultats/",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-a",
|
|
|
|
|
"--dossier-archive",
|
|
|
|
|
type=str,
|
|
|
|
|
help="Définir le dossier d'archive",
|
|
|
|
|
default="resultats/archives/",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
"-na",
|
|
|
|
|
"--no-archivage",
|
|
|
|
|
dest="archivage",
|
|
|
|
|
action="store_false",
|
|
|
|
|
help="Désactiver l'archivage'",
|
|
|
|
|
)
|
|
|
|
|
parser.set_defaults(archivage=True)
|
|
|
|
|
|
2021-10-22 22:09:41 +02:00
|
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
2021-10-10 17:55:26 +02:00
|
|
|
|
def main():
|
|
|
|
|
"""Routine principale"""
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-22 22:09:41 +02:00
|
|
|
|
parser = init_argparse()
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.basicConfig(
|
|
|
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
|
|
|
level=getattr(logging, args.log_level.upper()),
|
2021-10-23 19:56:20 +02:00
|
|
|
|
handlers=[logging.FileHandler("rdoo.log"), logging.StreamHandler(sys.stdout)],
|
2021-10-23 19:48:52 +02:00
|
|
|
|
)
|
|
|
|
|
|
2021-10-24 08:24:41 +02:00
|
|
|
|
utils = Utils(OVERPASS_URL, GEO_API_URL, args.dossier_resultats)
|
2021-10-23 22:03:25 +02:00
|
|
|
|
|
2021-10-24 08:24:41 +02:00
|
|
|
|
if args.archivage:
|
|
|
|
|
utils.archivage(args.dossier_archive)
|
2021-10-23 22:03:25 +02:00
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.info("##### Nouvelle récupération ######")
|
|
|
|
|
|
2021-10-22 22:09:41 +02:00
|
|
|
|
# l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM
|
|
|
|
|
aire_de_recherche = str(3_600_000_000 + args.zone)
|
|
|
|
|
|
2021-10-10 17:55:26 +02:00
|
|
|
|
for req in requetes.REQS:
|
|
|
|
|
for nb_essai in range(MAX_RETRY): # on tente max_retry fois
|
|
|
|
|
try:
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.info(f"# Requête en cours : {req.nom}")
|
2021-10-10 18:54:19 +02:00
|
|
|
|
|
2021-10-10 17:55:26 +02:00
|
|
|
|
# appel overpass
|
2021-10-22 22:09:41 +02:00
|
|
|
|
data = utils.run_overpass_query(req.critere, aire_de_recherche)
|
2021-10-10 19:24:18 +02:00
|
|
|
|
nb_resultats = len(data["elements"])
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.info(f"{nb_resultats} résultats")
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 19:24:18 +02:00
|
|
|
|
if nb_resultats > 0:
|
2021-10-22 22:35:56 +02:00
|
|
|
|
if args.geocoding_inverse:
|
|
|
|
|
# géocodage inverse
|
2021-11-01 18:58:42 +01:00
|
|
|
|
data = utils.geocodage_csv(data)
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 19:24:18 +02:00
|
|
|
|
# traduction
|
|
|
|
|
data = utils.traduction(
|
|
|
|
|
"bicycle_parking", TRAD_BICYCLE_PARKING, data
|
|
|
|
|
)
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 19:24:18 +02:00
|
|
|
|
# Sauvegarde
|
2021-10-24 08:24:41 +02:00
|
|
|
|
os.makedirs(args.dossier_resultats, exist_ok=True)
|
2021-10-10 19:24:18 +02:00
|
|
|
|
export_json = utils.nettoyage_json_pour_umap(data, req.champs)
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 19:24:18 +02:00
|
|
|
|
utils.save_as_json(export_json, req.nom)
|
|
|
|
|
utils.save_as_ods(req.champs, data, req.nom)
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
2021-10-10 19:53:43 +02:00
|
|
|
|
# doucement sur overpass
|
|
|
|
|
time.sleep(30)
|
2021-10-09 14:31:01 +02:00
|
|
|
|
break
|
2021-10-10 16:31:54 +02:00
|
|
|
|
except errors.ApiError:
|
2021-10-10 10:56:13 +02:00
|
|
|
|
|
2021-11-06 19:33:43 +01:00
|
|
|
|
if nb_essai >= MAX_RETRY - 1:
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.error("Trop d'erreurs d'API - abandon")
|
2021-11-06 19:33:43 +01:00
|
|
|
|
exit()
|
2021-10-10 10:56:13 +02:00
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.error(f"Erreur API - on retente dans {RETRY_DELAY}s")
|
2021-11-01 21:58:20 +01:00
|
|
|
|
except:
|
|
|
|
|
logging.error("Exception", stack_info=True, exc_info=True)
|
2021-10-10 10:56:13 +02:00
|
|
|
|
|
2021-10-10 16:50:27 +02:00
|
|
|
|
time.sleep(RETRY_DELAY)
|
2021-10-10 10:56:13 +02:00
|
|
|
|
|
2021-10-23 19:48:52 +02:00
|
|
|
|
logging.info("##### Terminé #####")
|
2021-09-26 12:52:37 +02:00
|
|
|
|
|
|
|
|
|
|
2021-10-09 14:31:01 +02:00
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|