encodage

explication sauvegarde ods
externalisation des requêtes et des champs génériques
2021-12-29 21:15:24 +01:00 · 2021-12-29 21:14:52 +01:00 · 2021-12-29 20:58:05 +01:00 · 2021-12-29 20:50:24 +01:00
7 changed files with 321 additions and 205 deletions
--- a/README.md
+++ b/README.md
@ -45,6 +45,9 @@ Ces chemins sont relatifs à `rdoo.py`, il est possible de passer des chemins ab

 Il est possible de ne pas archiver en passant l'argument `-na, --no-archive`.

+### Traductions
+Les tags peuvent être traduits grâce au fichier `traductions.json` contenant la configuration sous la forme `"clef" : "tableau_de_valeurs"` où la clef est la valeur du tag OSM à traduire et le tableau de valeurs est de la forme `"valeur" : "traduction"`. Le fichier par défaut contient l'exemple de la traduction du tag `"bicycle_parking"`.
+
 ## Umap
 Les fichiers json générés peuvent être directements utilisés dans [umap](https://umap.openstreetmap.fr/fr/) en les [important](https://wiki.openstreetmap.org/wiki/FR:UMap/Guide/Importer_un_fichier_de_donn%C3%A9es) et choisissant le format de données `osm`.

--- a/champs_generiques.json
+++ b/champs_generiques.json
@ -0,0 +1,148 @@
+{
+    "CHAMPS_STATIONNEMENT": {
+        "amenity": {
+            "export_json": "Non",
+            "FR": "aménagement"
+        },
+        "capacity": {
+            "export_json": "Oui",
+            "FR": "nombre d'emplacements"
+        },
+        "access": {
+            "export_json": "Oui",
+            "FR": "accès"
+        },
+        "bicycle_parking": {
+            "export_json": "Oui",
+            "FR": "type"
+        },
+        "covered": {
+            "export_json": "Oui",
+            "FR": "couvert"
+        },
+        "operator": {
+            "export_json": "Oui",
+            "FR": "opérateur"
+        },
+        "operator:type": {
+            "export_json": "Oui",
+            "FR": "type d'opérateur"
+        },
+        "fee": {
+            "export_json": "Oui",
+            "FR": "frais"
+        },
+        "check_date:capacity": {
+            "export_json": "Non",
+            "FR": "date_vérification"
+        },
+        "source": {
+            "export_json": "Non",
+            "FR": "source"
+        }
+    },
+    "CHAMPS_POI": {
+        "name": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "description": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "website": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "addr:housenumber": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "addr:street": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "addr:postcode": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "addr:city": {
+            "export_json": "Oui",
+            "FR": ""
+        },
+        "contact:email": {
+            "export_json": "Oui",
+            "FR": "email"
+        },
+        "contact:twitter": {
+            "export_json": "Oui",
+            "FR": "Twitter"
+        },
+        "contact:facebook": {
+            "export_json": "Oui",
+            "FR": "Facebook"
+        },
+        "contact:phone": {
+            "export_json": "Oui",
+            "FR": "Téléphone"
+        },
+        "network": {
+            "export_json": "Oui",
+            "FR": "Réseau"
+        },
+        "office": {
+            "export_json": "Oui",
+            "FR": "Bureau"
+        },
+        "opening_hours": {
+            "export_json": "Oui",
+            "FR": "Horaires"
+        }
+    },
+    "CHAMPS_ADRESSE": {
+        "api_adresse:geometry:coordinates:lon": {
+            "export_json": "Non",
+            "FR": "lon_adresse_etalab"
+        },
+        "api_adresse:geometry:coordinates:lat": {
+            "export_json": "Non",
+            "FR": "lat_adresse_etalab"
+        },
+        "api_adresse:properties:label": {
+            "export_json": "Non",
+            "FR": "adresse_etalab"
+        },
+        "api_adresse:properties:score": {
+            "export_json": "Non",
+            "FR": "score_etalab"
+        },
+        "api_adresse:properties:housenumber": {
+            "export_json": "Non",
+            "FR": "numero_etalab"
+        },
+        "api_adresse:properties:type": {
+            "export_json": "Non",
+            "FR": "type_etalab"
+        },
+        "api_adresse:properties:name": {
+            "export_json": "Non",
+            "FR": "numero_et_voie_etalab"
+        },
+        "api_adresse:properties:postcode": {
+            "export_json": "Non",
+            "FR": "code_postal_etalab"
+        },
+        "api_adresse:properties:citycode": {
+            "export_json": "Non",
+            "FR": "code_INSEE_etalab"
+        },
+        "api_adresse:properties:city": {
+            "export_json": "Non",
+            "FR": "ville_etalab"
+        },
+        "api_adresse:properties:street": {
+            "export_json": "Non",
+            "FR": "rue_etalab"
+        }
+    }
+}
--- a/osm_vc63/requetes.py
+++ b/osm_vc63/requetes.py
@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-
-
-# Copyright 2021 Olav63, SebF
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
-"""Module des requêtes"""
-
-
-class Requete:
-    """Objet requête"""
-
-    nom: str
-    critere: str
-    champs: dict
-
-    def __init__(self, nom, critere, champs):
-        self.nom = nom
-        self.critere = critere
-        self.champs = champs
-
-
-REQS = []
-CHAMPS_STATIONNEMENT = {
-    "amenity": {"export_json": "Non", "FR": "aménagement"},
-    "capacity": {"export_json": "Oui", "FR": "nombre d'emplacements"},
-    "access": {"export_json": "Oui", "FR": "accès"},
-    "bicycle_parking": {"export_json": "Oui", "FR": "type"},
-    "covered": {"export_json": "Oui", "FR": "couvert"},
-    "operator": {"export_json": "Oui", "FR": "opérateur"},
-    "operator:type": {"export_json": "Oui", "FR": "type d'opérateur"},
-    "fee": {"export_json": "Oui", "FR": "frais"},
-    "check_date:capacity": {"export_json": "Non", "FR": "date_vérification"},
-    "source": {"export_json": "Non", "FR": "source"},
-}
-CHAMPS_POI = {
-    "name": {"export_json": "Oui", "FR": ""},
-    "description": {"export_json": "Oui", "FR": ""},
-    "website": {"export_json": "Oui", "FR": ""},
-    "addr:housenumber": {"export_json": "Oui", "FR": ""},
-    "addr:street": {"export_json": "Oui", "FR": ""},
-    "addr:postcode": {"export_json": "Oui", "FR": ""},
-    "addr:city": {"export_json": "Oui", "FR": ""},
-    "contact:email": {"export_json": "Oui", "FR": "email"},
-    "contact:twitter": {"export_json": "Oui", "FR": "Twitter"},
-    "contact:facebook": {"export_json": "Oui", "FR": "Facebook"},
-    "contact:phone": {"export_json": "Oui", "FR": "Téléphone"},
-    "network": {"export_json": "Oui", "FR": "Réseau"},
-    "office": {"export_json": "Oui", "FR": "Bureau"},
-    "opening_hours": {"export_json": "Oui", "FR": "Horaires"},
-}
-CHAMPS_ADRESSE = {
-    "api_adresse:geometry:coordinates:lon": {
-        "export_json": "Non",
-        "FR": "lon_adresse_etalab",
-    },
-    "api_adresse:geometry:coordinates:lat": {
-        "export_json": "Non",
-        "FR": "lat_adresse_etalab",
-    },
-    "api_adresse:properties:label": {"export_json": "Non", "FR": "adresse_etalab"},
-    "api_adresse:properties:score": {"export_json": "Non", "FR": "score_etalab"},
-    "api_adresse:properties:housenumber": {"export_json": "Non", "FR": "numero_etalab"},
-    "api_adresse:properties:type": {"export_json": "Non", "FR": "type_etalab"},
-    "api_adresse:properties:name": {
-        "export_json": "Non",
-        "FR": "numero_et_voie_etalab",
-    },
-    "api_adresse:properties:postcode": {
-        "export_json": "Non",
-        "FR": "code_postal_etalab",
-    },
-    "api_adresse:properties:citycode": {
-        "export_json": "Non",
-        "FR": "code_INSEE_etalab",
-    },
-    "api_adresse:properties:city": {"export_json": "Non", "FR": "ville_etalab"},
-    "api_adresse:properties:street": {"export_json": "Non", "FR": "rue_etalab"},
-}
-
-# pylint: disable=C0301
-STATIONNEMENT_NON_PUBLIC_REQ = r'nwr["amenity"="bicycle_parking"]["access"~"(no|permit|private|customers|permissive)"](area:aire_de_recherche);'
-REQS.append(
-    Requete(
-        "stationnements_velos_non_publics",
-        STATIONNEMENT_NON_PUBLIC_REQ,
-        dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE),
-    )
-)
-
-REQS.append(
-    Requete(
-        "stationnements_velos_publics",
-        # pylint: disable=C0301
-        rf'nwr["amenity"="bicycle_parking"](area:aire_de_recherche); - {STATIONNEMENT_NON_PUBLIC_REQ}',
-        dict(CHAMPS_STATIONNEMENT, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"service:bicycle:diy": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "ateliers_autoreparation",
-        r'nwr["service:bicycle:diy"="yes"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"association": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "associations_velo",
-        r'nwr["association"="bicycle"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"craft": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "fabriquants_velo",
-        r'nwr["craft"="bicycle"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"shop": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "vendeurs_velo",
-        # pylint: disable=C0301
-        r'nwr["shop"="bicycle"](area:aire_de_recherche); nwr["service:bicycle:retail"="yes"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"amenity": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "velos_libre_service",
-        r'nwr["amenity"="bicycle_rental"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
-
-CHAMP_LOCAL = {"service:bicycle:rental": {"export_json": "Non", "FR": ""}}
-REQS.append(
-    Requete(
-        "location_velo",
-        r'nwr["service:bicycle:rental"="yes"](area:aire_de_recherche);',
-        dict(CHAMP_LOCAL, **CHAMPS_POI, **CHAMPS_ADRESSE),
-    )
-)
--- a/osm_vc63/utils.py
+++ b/osm_vc63/utils.py
@ -44,8 +44,17 @@ class Utils:
        self.geo_api_url = geo_api_url
        self.dossier_resultats = dossier_resultats

+        with open("traductions.json", encoding="utf-8") as trads:
+            self.traductions = json.load(trads)
+
+        self.lecture_requetes()
+
    def save_as_ods(self, fields, data, nom_req, ods_data_sheet=OrderedDict()):
-        """Sauvegarde de data dans un classeur ods"""
+        """Sauvegarde de data dans un classeur ods.
+
+        Le paramètre ods_data_sheet est évalué une seule fois à la définition de la fonction,
+        ce qui enregistre les data de chaque appel dans une nouvelle feuille.
+        """

        # ods_data_sheet = OrderedDict()
        ods_data = []
@ -79,7 +88,9 @@ class Utils:
    def save_as_json(self, export_json, nom_req):
        """Enregistrement du JSON"""

-        json_file = open(self.dossier_resultats + nom_req + ".json", "w")
+        json_file = open(
+            self.dossier_resultats + nom_req + ".json", "w", encoding="utf-8"
+        )
        json_file.write(json.dumps(export_json))
        json_file.close()

@ -158,7 +169,9 @@ class Utils:
        url = self.geo_api_url + "/reverse/csv/"

        # création du fichier à envoyer à l'API
-        with open("tmp_geocodage.csv", "w", newline="") as tmp_csv_file:
+        with open(
+            "tmp_geocodage.csv", "w", newline="", encoding="utf-8"
+        ) as tmp_csv_file:
            csv_writer = csv.writer(tmp_csv_file)
            csv_writer.writerow(["lat", "lon"])

@ -254,3 +267,21 @@ class Utils:
            for file in os.listdir(self.dossier_resultats)
            if not os.path.isdir(self.dossier_resultats + file)
        ]
+
+    def lecture_requetes(self):
+        """Lecture des requêtes dans les fichiers de configuration"""
+
+        with open("requetes.json", encoding="utf-8") as reqs:
+            self.json_reqs = json.load(reqs)
+
+        with open("champs_generiques.json", encoding="utf-8") as champs_generiques:
+            self.json_champs_generiques = json.load(champs_generiques)
+
+        for req in self.json_reqs:
+            self.json_reqs[req]["champs"] = dict(self.json_reqs[req]["champ_local"])
+            for champ in self.json_reqs[req]["champs_generiques"]:
+                self.json_reqs[req]["champs"].update(self.json_champs_generiques[champ])
+
+            # nettoyage
+            self.json_reqs[req].pop("champ_local", None)
+            self.json_reqs[req].pop("champs_generiques", None)
--- a/rdoo.py
+++ b/rdoo.py
@ -18,7 +18,7 @@


 """
-Module principal : 
+Module principal :
 - récupération de données par appel à Overpass
 - géocodage inverse
 - export des données en JSON pour utilisation avec umap
@ -32,7 +32,6 @@ import argparse
 import logging
 import sys
 from osm_vc63 import errors
-from osm_vc63 import requetes
 from osm_vc63.utils import Utils

 OVERPASS_URL = "http://overpass-api.de/api/interpreter"
@ -45,30 +44,6 @@ MAX_RETRY = 4
 RETRY_DELAY = 120


-# traductions des tags bicycle_parking
-TRAD_BICYCLE_PARKING = {
-    "stands": "Arceaux",
-    "wall_loops": "Pince roues",
-    "rack": "Râteliers",
-    "anchors": "Ancrage",
-    "shed": "Abri collectif",
-    "bollard": "Potelet",
-    "lockers": "Abris individuels",
-    "wide_stands": "Arceaux espacés",
-    "ground_slots": "Fente dans le sol",
-    "building": "Bâtiment",
-    "informal": "Informel",
-    "wave": "Râteliers",
-    "streetpod": "Arceaux",
-    "tree": "Arbre à bicyclettes",
-    "crossbar": "Barre",
-    "rope": "Câble",
-    "two-tier": "Deux étages",
-    "floor": "Sol",
-    "handlebar_holder": "Accroche-guidons",
-}
-
-
 def init_argparse() -> argparse.ArgumentParser:
    """Définition des arguments possibles."""

@ -142,7 +117,10 @@ def main():
    logging.basicConfig(
        format="%(asctime)s [%(levelname)s] %(message)s",
        level=getattr(logging, args.log_level.upper()),
-        handlers=[logging.FileHandler("rdoo.log"), logging.StreamHandler(sys.stdout)],
+        handlers=[
+            logging.FileHandler("rdoo.log", encoding="utf-8"),
+            logging.StreamHandler(sys.stdout),
+        ],
    )

    utils = Utils(OVERPASS_URL, GEO_API_URL, args.dossier_resultats)
@ -155,14 +133,16 @@ def main():
    # l'id de l'area se calcule en ajoutant 3600000000 au numéro de l'objet OSM
    aire_de_recherche = str(3_600_000_000 + args.zone)

-    for req in requetes.REQS:
+    for req in utils.json_reqs:
        for nb_essai in range(MAX_RETRY):  # on tente max_retry fois
            try:

-                logging.info(f"# Requête en cours : {req.nom}")
+                logging.info(f"# Requête en cours : {req}")

                # appel overpass
-                data = utils.run_overpass_query(req.critere, aire_de_recherche)
+                data = utils.run_overpass_query(
+                    utils.json_reqs[req]["overpass"], aire_de_recherche
+                )
                nb_resultats = len(data["elements"])
                logging.info(f"{nb_resultats} résultats")

@ -172,16 +152,17 @@ def main():
                        data = utils.geocodage_csv(data)

                    # traduction
-                    data = utils.traduction(
-                        "bicycle_parking", TRAD_BICYCLE_PARKING, data
-                    )
+                    for key in utils.traductions:
+                        data = utils.traduction(key, utils.traductions[key], data)

                    # Sauvegarde
                    os.makedirs(args.dossier_resultats, exist_ok=True)
-                    export_json = utils.nettoyage_json_pour_umap(data, req.champs)
+                    export_json = utils.nettoyage_json_pour_umap(
+                        data, utils.json_reqs[req]["champs"]
+                    )

-                    utils.save_as_json(export_json, req.nom)
-                    utils.save_as_ods(req.champs, data, req.nom)
+                    utils.save_as_json(export_json, req)
+                    utils.save_as_ods(utils.json_reqs[req]["champs"], data, req)

                # doucement sur overpass
                time.sleep(30)
--- a/requetes.json
+++ b/requetes.json
@ -0,0 +1,96 @@
+{
+    "stationnements_velos_non_publics": {
+        "overpass": "nwr[\"amenity\"=\"bicycle_parking\"][\"access\"~\"(no|permit|private|customers|permissive)\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_STATIONNEMENT",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": ""
+    },
+    "stationnements_velos_publics": {
+        "overpass": "nwr[\"amenity\"=\"bicycle_parking\"](area:aire_de_recherche); - nwr[\"amenity\"=\"bicycle_parking\"][\"access\"~\"(no|permit|private|customers|permissive)\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_STATIONNEMENT",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": ""
+    },
+    "ateliers_autoreparation": {
+        "overpass": "nwr[\"service:bicycle:diy\"=\"yes\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "service:bicycle:diy": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    },
+    "associations_velo": {
+        "overpass": "nwr[\"association\"=\"bicycle\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "association": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    },
+    "fabriquants_velo": {
+        "overpass": "nwr[\"craft\"=\"bicycle\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "craft": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    },
+    "vendeurs_velo": {
+        "overpass": "nwr[\"shop\"=\"bicycle\"](area:aire_de_recherche); nwr[\"service:bicycle:retail\"=\"yes\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "shop": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    },
+    "velos_libre_service": {
+        "overpass": "nwr[\"amenity\"=\"bicycle_rental\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "amenity": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    },
+    "location_velo": {
+        "overpass": "nwr[\"service:bicycle:rental\"=\"yes\"](area:aire_de_recherche);",
+        "champs_generiques": [
+            "CHAMPS_POI",
+            "CHAMPS_ADRESSE"
+        ],
+        "champ_local": {
+            "service:bicycle:rental": {
+                "export_json": "Non",
+                "FR": ""
+            }
+        }
+    }
+}
--- a/traductions.json
+++ b/traductions.json
@ -0,0 +1,23 @@
+{
+    "bicycle_parking": {
+        "stands": "Arceaux",
+        "wall_loops": "Pince roues",
+        "rack": "Râteliers",
+        "anchors": "Ancrage",
+        "shed": "Abri collectif",
+        "bollard": "Potelet",
+        "lockers": "Abris individuels",
+        "wide_stands": "Arceaux espacés",
+        "ground_slots": "Fente dans le sol",
+        "building": "Bâtiment",
+        "informal": "Informel",
+        "wave": "Râteliers",
+        "streetpod": "Arceaux",
+        "tree": "Arbre à bicyclettes",
+        "crossbar": "Barre",
+        "rope": "Câble",
+        "two-tier": "Deux étages",
+        "floor": "Sol",
+        "handlebar_holder": "Accroche-guidons"
+    }
+}
Author	SHA1	Message	Date
SebF	1c0421f87d	encodage	2021-12-29 21:15:24 +01:00
SebF	37273e4953	explication sauvegarde ods	2021-12-29 21:14:52 +01:00
SebF	4960e78fa8	externalisation des requêtes et des champs génériques	2021-12-29 20:58:05 +01:00
SebF	ee7aebf5fb	configuration des traductions	2021-12-29 20:50:24 +01:00