RDOO/rdoo.py

184 lines
5.1 KiB
Python
Raw Normal View History

2021-09-26 12:52:37 +02:00
#!/usr/bin/env python3
2021-10-21 22:27:19 +02:00
# Copyright 2021 Olav63, SebF
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
2021-10-10 16:50:27 +02:00
"""
Module principal : 
- récupération de données par appel à Overpass
- géocodage inverse
- export des données en JSON pour utilisation avec umap
- sauvegarde des données en ods
"""
2021-09-26 12:52:37 +02:00
import time
import os
import argparse
2021-10-23 19:48:52 +02:00
import logging
import sys
from osm_vc63 import errors
from osm_vc63.utils import Utils
2021-09-26 12:52:37 +02:00
2021-10-10 16:50:27 +02:00
OVERPASS_URL = "http://overpass-api.de/api/interpreter"
GEO_API_URL = "https://api-adresse.data.gouv.fr"
2021-09-26 12:52:37 +02:00
# nombre maxi de retries quand echec API
2021-10-10 16:50:27 +02:00
MAX_RETRY = 4
2021-09-26 12:52:37 +02:00
# delai en secondes entre les tentatives
2021-10-10 16:50:27 +02:00
RETRY_DELAY = 120
2021-09-26 12:52:37 +02:00
def init_argparse() -> argparse.ArgumentParser:
"""Définition des arguments possibles."""
parser = argparse.ArgumentParser(
usage="%(prog)s [OPTIONS] ...",
description="Exporte les données de la Cyclosphère d'une zone géographique.",
)
parser.add_argument(
"-z",
"--zone",
type=int,
2021-10-23 19:48:52 +02:00
help="Choisir la zone géographique à inspecter.",
default=7406,
)
parser.add_argument(
"-gi",
"--geocoding-inverse",
dest="geocoding_inverse",
action="store_true",
help="Activer le géocodage inverse",
)
parser.add_argument(
"-ngi",
"--no-geocoding-inverse",
dest="geocoding_inverse",
action="store_false",
help="Désactiver le géocodage inverse",
)
parser.set_defaults(geocoding_inverse=True)
2021-10-23 19:48:52 +02:00
parser.add_argument(
2021-10-23 22:11:11 +02:00
"-l", "--log-level", type=str, help="Définir le niveau de log.", default="INFO"
2021-10-23 19:48:52 +02:00
)
parser.add_argument(
"-r",
"--dossier-resultats",
type=str,
help="Définir le dossier d'archive",
default="resultats/",
)
parser.add_argument(
"-a",
"--dossier-archive",
type=str,
help="Définir le dossier d'archive",
default="resultats/archives/",
)
parser.add_argument(
"-na",
"--no-archivage",
dest="archivage",
action="store_false",
help="Désactiver l'archivage'",
)
parser.set_defaults(archivage=True)
return parser
def main():
"""Routine principale"""
2021-09-26 12:52:37 +02:00
parser = init_argparse()
args = parser.parse_args()
2021-10-23 19:48:52 +02:00
logging.basicConfig(
format="%(asctime)s [%(levelname)s] %(message)s",
level=getattr(logging, args.log_level.upper()),
2021-10-23 19:56:20 +02:00
handlers=[logging.FileHandler("rdoo.log"), logging.StreamHandler(sys.stdout)],
2021-10-23 19:48:52 +02:00
)
utils = Utils(OVERPASS_URL, GEO_API_URL, args.dossier_resultats)
2021-10-23 22:03:25 +02:00
if args.archivage:
utils.archivage(args.dossier_archive)
2021-10-23 22:03:25 +02:00
2021-10-23 19:48:52 +02:00
logging.info("##### Nouvelle récupération ######")
# l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM
aire_de_recherche = str(3_600_000_000 + args.zone)
for req in utils.json_reqs:
for nb_essai in range(MAX_RETRY): # on tente max_retry fois
try:
2021-09-26 12:52:37 +02:00
logging.info(f"# Requête en cours : {req}")
2021-10-10 18:54:19 +02:00
# appel overpass
data = utils.run_overpass_query(
utils.json_reqs[req]["overpass"], aire_de_recherche
)
nb_resultats = len(data["elements"])
2021-10-23 19:48:52 +02:00
logging.info(f"{nb_resultats} résultats")
2021-09-26 12:52:37 +02:00
if nb_resultats > 0:
if args.geocoding_inverse:
# géocodage inverse
data = utils.geocodage_csv(data)
2021-09-26 12:52:37 +02:00
# traduction
2021-12-29 20:50:24 +01:00
for key in utils.traductions:
data = utils.traduction(key, utils.traductions[key], data)
2021-09-26 12:52:37 +02:00
# Sauvegarde
os.makedirs(args.dossier_resultats, exist_ok=True)
export_json = utils.nettoyage_json_pour_umap(
data, utils.json_reqs[req]["champs"]
)
2021-09-26 12:52:37 +02:00
utils.save_as_json(export_json, req)
utils.save_as_ods(utils.json_reqs[req]["champs"], data, req)
2021-09-26 12:52:37 +02:00
2021-10-10 19:53:43 +02:00
# doucement sur overpass
time.sleep(30)
2021-10-09 14:31:01 +02:00
break
2021-10-10 16:31:54 +02:00
except errors.ApiError:
2021-10-10 10:56:13 +02:00
2021-11-06 19:33:43 +01:00
if nb_essai >= MAX_RETRY - 1:
2021-10-23 19:48:52 +02:00
logging.error("Trop d'erreurs d'API - abandon")
2021-11-06 19:33:43 +01:00
exit()
2021-10-10 10:56:13 +02:00
2021-10-23 19:48:52 +02:00
logging.error(f"Erreur API - on retente dans {RETRY_DELAY}s")
2021-11-01 21:58:20 +01:00
except:
logging.error("Exception", stack_info=True, exc_info=True)
2021-10-10 10:56:13 +02:00
2021-10-10 16:50:27 +02:00
time.sleep(RETRY_DELAY)
2021-10-10 10:56:13 +02:00
2021-10-23 19:48:52 +02:00
logging.info("##### Terminé #####")
2021-09-26 12:52:37 +02:00
2021-10-09 14:31:01 +02:00
if __name__ == "__main__":
main()